1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020 Western Digital Corporation or its affiliates. 4 */ 5 #include <linux/kernel.h> 6 #include <linux/init.h> 7 #include <linux/mm.h> 8 #include <linux/module.h> 9 #include <linux/perf_event.h> 10 #include <linux/irq.h> 11 #include <linux/stringify.h> 12 13 #include <asm/processor.h> 14 #include <asm/ptrace.h> 15 #include <asm/csr.h> 16 #include <asm/entry-common.h> 17 #include <asm/hwprobe.h> 18 #include <asm/cpufeature.h> 19 #include <asm/sbi.h> 20 #include <asm/vector.h> 21 22 #define INSN_MATCH_LB 0x3 23 #define INSN_MASK_LB 0x707f 24 #define INSN_MATCH_LH 0x1003 25 #define INSN_MASK_LH 0x707f 26 #define INSN_MATCH_LW 0x2003 27 #define INSN_MASK_LW 0x707f 28 #define INSN_MATCH_LD 0x3003 29 #define INSN_MASK_LD 0x707f 30 #define INSN_MATCH_LBU 0x4003 31 #define INSN_MASK_LBU 0x707f 32 #define INSN_MATCH_LHU 0x5003 33 #define INSN_MASK_LHU 0x707f 34 #define INSN_MATCH_LWU 0x6003 35 #define INSN_MASK_LWU 0x707f 36 #define INSN_MATCH_SB 0x23 37 #define INSN_MASK_SB 0x707f 38 #define INSN_MATCH_SH 0x1023 39 #define INSN_MASK_SH 0x707f 40 #define INSN_MATCH_SW 0x2023 41 #define INSN_MASK_SW 0x707f 42 #define INSN_MATCH_SD 0x3023 43 #define INSN_MASK_SD 0x707f 44 45 #define INSN_MATCH_FLW 0x2007 46 #define INSN_MASK_FLW 0x707f 47 #define INSN_MATCH_FLD 0x3007 48 #define INSN_MASK_FLD 0x707f 49 #define INSN_MATCH_FLQ 0x4007 50 #define INSN_MASK_FLQ 0x707f 51 #define INSN_MATCH_FSW 0x2027 52 #define INSN_MASK_FSW 0x707f 53 #define INSN_MATCH_FSD 0x3027 54 #define INSN_MASK_FSD 0x707f 55 #define INSN_MATCH_FSQ 0x4027 56 #define INSN_MASK_FSQ 0x707f 57 58 #define INSN_MATCH_C_LD 0x6000 59 #define INSN_MASK_C_LD 0xe003 60 #define INSN_MATCH_C_SD 0xe000 61 #define INSN_MASK_C_SD 0xe003 62 #define INSN_MATCH_C_LW 0x4000 63 #define INSN_MASK_C_LW 0xe003 64 #define INSN_MATCH_C_SW 0xc000 65 #define INSN_MASK_C_SW 0xe003 66 #define INSN_MATCH_C_LDSP 0x6002 67 #define INSN_MASK_C_LDSP 0xe003 68 #define INSN_MATCH_C_SDSP 0xe002 69 #define INSN_MASK_C_SDSP 0xe003 70 #define INSN_MATCH_C_LWSP 0x4002 71 #define INSN_MASK_C_LWSP 0xe003 72 #define INSN_MATCH_C_SWSP 0xc002 73 #define INSN_MASK_C_SWSP 0xe003 74 75 #define INSN_MATCH_C_FLD 0x2000 76 #define INSN_MASK_C_FLD 0xe003 77 #define INSN_MATCH_C_FLW 0x6000 78 #define INSN_MASK_C_FLW 0xe003 79 #define INSN_MATCH_C_FSD 0xa000 80 #define INSN_MASK_C_FSD 0xe003 81 #define INSN_MATCH_C_FSW 0xe000 82 #define INSN_MASK_C_FSW 0xe003 83 #define INSN_MATCH_C_FLDSP 0x2002 84 #define INSN_MASK_C_FLDSP 0xe003 85 #define INSN_MATCH_C_FSDSP 0xa002 86 #define INSN_MASK_C_FSDSP 0xe003 87 #define INSN_MATCH_C_FLWSP 0x6002 88 #define INSN_MASK_C_FLWSP 0xe003 89 #define INSN_MATCH_C_FSWSP 0xe002 90 #define INSN_MASK_C_FSWSP 0xe003 91 92 #define INSN_MATCH_C_LHU 0x8400 93 #define INSN_MASK_C_LHU 0xfc43 94 #define INSN_MATCH_C_LH 0x8440 95 #define INSN_MASK_C_LH 0xfc43 96 #define INSN_MATCH_C_SH 0x8c00 97 #define INSN_MASK_C_SH 0xfc43 98 99 #define INSN_LEN(insn) ((((insn) & 0x3) < 0x3) ? 2 : 4) 100 101 #if defined(CONFIG_64BIT) 102 #define LOG_REGBYTES 3 103 #define XLEN 64 104 #else 105 #define LOG_REGBYTES 2 106 #define XLEN 32 107 #endif 108 #define REGBYTES (1 << LOG_REGBYTES) 109 #define XLEN_MINUS_16 ((XLEN) - 16) 110 111 #define SH_RD 7 112 #define SH_RS1 15 113 #define SH_RS2 20 114 #define SH_RS2C 2 115 116 #define RV_X(x, s, n) (((x) >> (s)) & ((1 << (n)) - 1)) 117 #define RVC_LW_IMM(x) ((RV_X(x, 6, 1) << 2) | \ 118 (RV_X(x, 10, 3) << 3) | \ 119 (RV_X(x, 5, 1) << 6)) 120 #define RVC_LD_IMM(x) ((RV_X(x, 10, 3) << 3) | \ 121 (RV_X(x, 5, 2) << 6)) 122 #define RVC_LWSP_IMM(x) ((RV_X(x, 4, 3) << 2) | \ 123 (RV_X(x, 12, 1) << 5) | \ 124 (RV_X(x, 2, 2) << 6)) 125 #define RVC_LDSP_IMM(x) ((RV_X(x, 5, 2) << 3) | \ 126 (RV_X(x, 12, 1) << 5) | \ 127 (RV_X(x, 2, 3) << 6)) 128 #define RVC_SWSP_IMM(x) ((RV_X(x, 9, 4) << 2) | \ 129 (RV_X(x, 7, 2) << 6)) 130 #define RVC_SDSP_IMM(x) ((RV_X(x, 10, 3) << 3) | \ 131 (RV_X(x, 7, 3) << 6)) 132 #define RVC_RS1S(insn) (8 + RV_X(insn, SH_RD, 3)) 133 #define RVC_RS2S(insn) (8 + RV_X(insn, SH_RS2C, 3)) 134 #define RVC_RS2(insn) RV_X(insn, SH_RS2C, 5) 135 136 #define SHIFT_RIGHT(x, y) \ 137 ((y) < 0 ? ((x) << -(y)) : ((x) >> (y))) 138 139 #define REG_MASK \ 140 ((1 << (5 + LOG_REGBYTES)) - (1 << LOG_REGBYTES)) 141 142 #define REG_OFFSET(insn, pos) \ 143 (SHIFT_RIGHT((insn), (pos) - LOG_REGBYTES) & REG_MASK) 144 145 #define REG_PTR(insn, pos, regs) \ 146 (ulong *)((ulong)(regs) + REG_OFFSET(insn, pos)) 147 148 #define GET_RS1(insn, regs) (*REG_PTR(insn, SH_RS1, regs)) 149 #define GET_RS2(insn, regs) (*REG_PTR(insn, SH_RS2, regs)) 150 #define GET_RS1S(insn, regs) (*REG_PTR(RVC_RS1S(insn), 0, regs)) 151 #define GET_RS2S(insn, regs) (*REG_PTR(RVC_RS2S(insn), 0, regs)) 152 #define GET_RS2C(insn, regs) (*REG_PTR(insn, SH_RS2C, regs)) 153 #define GET_SP(regs) (*REG_PTR(2, 0, regs)) 154 #define SET_RD(insn, regs, val) (*REG_PTR(insn, SH_RD, regs) = (val)) 155 #define IMM_I(insn) ((s32)(insn) >> 20) 156 #define IMM_S(insn) (((s32)(insn) >> 25 << 5) | \ 157 (s32)(((insn) >> 7) & 0x1f)) 158 #define MASK_FUNCT3 0x7000 159 160 #define GET_PRECISION(insn) (((insn) >> 25) & 3) 161 #define GET_RM(insn) (((insn) >> 12) & 7) 162 #define PRECISION_S 0 163 #define PRECISION_D 1 164 165 #ifdef CONFIG_FPU 166 167 #define FP_GET_RD(insn) (insn >> 7 & 0x1F) 168 169 extern void put_f32_reg(unsigned long fp_reg, unsigned long value); 170 171 static int set_f32_rd(unsigned long insn, struct pt_regs *regs, 172 unsigned long val) 173 { 174 unsigned long fp_reg = FP_GET_RD(insn); 175 176 put_f32_reg(fp_reg, val); 177 regs->status |= SR_FS_DIRTY; 178 179 return 0; 180 } 181 182 extern void put_f64_reg(unsigned long fp_reg, unsigned long value); 183 184 static int set_f64_rd(unsigned long insn, struct pt_regs *regs, u64 val) 185 { 186 unsigned long fp_reg = FP_GET_RD(insn); 187 unsigned long value; 188 189 #if __riscv_xlen == 32 190 value = (unsigned long) &val; 191 #else 192 value = val; 193 #endif 194 put_f64_reg(fp_reg, value); 195 regs->status |= SR_FS_DIRTY; 196 197 return 0; 198 } 199 200 #if __riscv_xlen == 32 201 extern void get_f64_reg(unsigned long fp_reg, u64 *value); 202 203 static u64 get_f64_rs(unsigned long insn, u8 fp_reg_offset, 204 struct pt_regs *regs) 205 { 206 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F; 207 u64 val; 208 209 get_f64_reg(fp_reg, &val); 210 regs->status |= SR_FS_DIRTY; 211 212 return val; 213 } 214 #else 215 216 extern unsigned long get_f64_reg(unsigned long fp_reg); 217 218 static unsigned long get_f64_rs(unsigned long insn, u8 fp_reg_offset, 219 struct pt_regs *regs) 220 { 221 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F; 222 unsigned long val; 223 224 val = get_f64_reg(fp_reg); 225 regs->status |= SR_FS_DIRTY; 226 227 return val; 228 } 229 230 #endif 231 232 extern unsigned long get_f32_reg(unsigned long fp_reg); 233 234 static unsigned long get_f32_rs(unsigned long insn, u8 fp_reg_offset, 235 struct pt_regs *regs) 236 { 237 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F; 238 unsigned long val; 239 240 val = get_f32_reg(fp_reg); 241 regs->status |= SR_FS_DIRTY; 242 243 return val; 244 } 245 246 #else /* CONFIG_FPU */ 247 static void set_f32_rd(unsigned long insn, struct pt_regs *regs, 248 unsigned long val) {} 249 250 static void set_f64_rd(unsigned long insn, struct pt_regs *regs, u64 val) {} 251 252 static unsigned long get_f64_rs(unsigned long insn, u8 fp_reg_offset, 253 struct pt_regs *regs) 254 { 255 return 0; 256 } 257 258 static unsigned long get_f32_rs(unsigned long insn, u8 fp_reg_offset, 259 struct pt_regs *regs) 260 { 261 return 0; 262 } 263 264 #endif 265 266 #define GET_F64_RS2(insn, regs) (get_f64_rs(insn, 20, regs)) 267 #define GET_F64_RS2C(insn, regs) (get_f64_rs(insn, 2, regs)) 268 #define GET_F64_RS2S(insn, regs) (get_f64_rs(RVC_RS2S(insn), 0, regs)) 269 270 #define GET_F32_RS2(insn, regs) (get_f32_rs(insn, 20, regs)) 271 #define GET_F32_RS2C(insn, regs) (get_f32_rs(insn, 2, regs)) 272 #define GET_F32_RS2S(insn, regs) (get_f32_rs(RVC_RS2S(insn), 0, regs)) 273 274 #define __read_insn(regs, insn, insn_addr, type) \ 275 ({ \ 276 int __ret; \ 277 \ 278 if (user_mode(regs)) { \ 279 __ret = get_user(insn, (type __user *) insn_addr); \ 280 } else { \ 281 insn = *(type *)insn_addr; \ 282 __ret = 0; \ 283 } \ 284 \ 285 __ret; \ 286 }) 287 288 static inline int get_insn(struct pt_regs *regs, ulong epc, ulong *r_insn) 289 { 290 ulong insn = 0; 291 292 if (epc & 0x2) { 293 ulong tmp = 0; 294 295 if (__read_insn(regs, insn, epc, u16)) 296 return -EFAULT; 297 /* __get_user() uses regular "lw" which sign extend the loaded 298 * value make sure to clear higher order bits in case we "or" it 299 * below with the upper 16 bits half. 300 */ 301 insn &= GENMASK(15, 0); 302 if ((insn & __INSN_LENGTH_MASK) != __INSN_LENGTH_32) { 303 *r_insn = insn; 304 return 0; 305 } 306 epc += sizeof(u16); 307 if (__read_insn(regs, tmp, epc, u16)) 308 return -EFAULT; 309 *r_insn = (tmp << 16) | insn; 310 311 return 0; 312 } else { 313 if (__read_insn(regs, insn, epc, u32)) 314 return -EFAULT; 315 if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32) { 316 *r_insn = insn; 317 return 0; 318 } 319 insn &= GENMASK(15, 0); 320 *r_insn = insn; 321 322 return 0; 323 } 324 } 325 326 union reg_data { 327 u8 data_bytes[8]; 328 ulong data_ulong; 329 u64 data_u64; 330 }; 331 332 /* sysctl hooks */ 333 int unaligned_enabled __read_mostly = 1; /* Enabled by default */ 334 335 #ifdef CONFIG_RISCV_VECTOR_MISALIGNED 336 static int handle_vector_misaligned_load(struct pt_regs *regs) 337 { 338 unsigned long epc = regs->epc; 339 unsigned long insn; 340 341 if (get_insn(regs, epc, &insn)) 342 return -1; 343 344 /* Only return 0 when in check_vector_unaligned_access_emulated */ 345 if (*this_cpu_ptr(&vector_misaligned_access) == RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) { 346 *this_cpu_ptr(&vector_misaligned_access) = RISCV_HWPROBE_MISALIGNED_VECTOR_UNSUPPORTED; 347 regs->epc = epc + INSN_LEN(insn); 348 return 0; 349 } 350 351 /* If vector instruction we don't emulate it yet */ 352 regs->epc = epc; 353 return -1; 354 } 355 #else 356 static int handle_vector_misaligned_load(struct pt_regs *regs) 357 { 358 return -1; 359 } 360 #endif 361 362 static int handle_scalar_misaligned_load(struct pt_regs *regs) 363 { 364 union reg_data val; 365 unsigned long epc = regs->epc; 366 unsigned long insn; 367 unsigned long addr = regs->badaddr; 368 int fp = 0, shift = 0, len = 0; 369 370 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr); 371 372 *this_cpu_ptr(&misaligned_access_speed) = RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED; 373 374 if (!unaligned_enabled) 375 return -1; 376 377 if (user_mode(regs) && (current->thread.align_ctl & PR_UNALIGN_SIGBUS)) 378 return -1; 379 380 if (get_insn(regs, epc, &insn)) 381 return -1; 382 383 regs->epc = 0; 384 385 if ((insn & INSN_MASK_LW) == INSN_MATCH_LW) { 386 len = 4; 387 shift = 8 * (sizeof(unsigned long) - len); 388 #if defined(CONFIG_64BIT) 389 } else if ((insn & INSN_MASK_LD) == INSN_MATCH_LD) { 390 len = 8; 391 shift = 8 * (sizeof(unsigned long) - len); 392 } else if ((insn & INSN_MASK_LWU) == INSN_MATCH_LWU) { 393 len = 4; 394 #endif 395 } else if ((insn & INSN_MASK_FLD) == INSN_MATCH_FLD) { 396 fp = 1; 397 len = 8; 398 } else if ((insn & INSN_MASK_FLW) == INSN_MATCH_FLW) { 399 fp = 1; 400 len = 4; 401 } else if ((insn & INSN_MASK_LH) == INSN_MATCH_LH) { 402 len = 2; 403 shift = 8 * (sizeof(unsigned long) - len); 404 } else if ((insn & INSN_MASK_LHU) == INSN_MATCH_LHU) { 405 len = 2; 406 #if defined(CONFIG_64BIT) 407 } else if ((insn & INSN_MASK_C_LD) == INSN_MATCH_C_LD) { 408 len = 8; 409 shift = 8 * (sizeof(unsigned long) - len); 410 insn = RVC_RS2S(insn) << SH_RD; 411 } else if ((insn & INSN_MASK_C_LDSP) == INSN_MATCH_C_LDSP && 412 ((insn >> SH_RD) & 0x1f)) { 413 len = 8; 414 shift = 8 * (sizeof(unsigned long) - len); 415 #endif 416 } else if ((insn & INSN_MASK_C_LW) == INSN_MATCH_C_LW) { 417 len = 4; 418 shift = 8 * (sizeof(unsigned long) - len); 419 insn = RVC_RS2S(insn) << SH_RD; 420 } else if ((insn & INSN_MASK_C_LWSP) == INSN_MATCH_C_LWSP && 421 ((insn >> SH_RD) & 0x1f)) { 422 len = 4; 423 shift = 8 * (sizeof(unsigned long) - len); 424 } else if ((insn & INSN_MASK_C_FLD) == INSN_MATCH_C_FLD) { 425 fp = 1; 426 len = 8; 427 insn = RVC_RS2S(insn) << SH_RD; 428 } else if ((insn & INSN_MASK_C_FLDSP) == INSN_MATCH_C_FLDSP) { 429 fp = 1; 430 len = 8; 431 #if defined(CONFIG_32BIT) 432 } else if ((insn & INSN_MASK_C_FLW) == INSN_MATCH_C_FLW) { 433 fp = 1; 434 len = 4; 435 insn = RVC_RS2S(insn) << SH_RD; 436 } else if ((insn & INSN_MASK_C_FLWSP) == INSN_MATCH_C_FLWSP) { 437 fp = 1; 438 len = 4; 439 #endif 440 } else if ((insn & INSN_MASK_C_LHU) == INSN_MATCH_C_LHU) { 441 len = 2; 442 insn = RVC_RS2S(insn) << SH_RD; 443 } else if ((insn & INSN_MASK_C_LH) == INSN_MATCH_C_LH) { 444 len = 2; 445 shift = 8 * (sizeof(ulong) - len); 446 insn = RVC_RS2S(insn) << SH_RD; 447 } else { 448 regs->epc = epc; 449 return -1; 450 } 451 452 if (!IS_ENABLED(CONFIG_FPU) && fp) 453 return -EOPNOTSUPP; 454 455 val.data_u64 = 0; 456 if (user_mode(regs)) { 457 if (copy_from_user(&val, (u8 __user *)addr, len)) 458 return -1; 459 } else { 460 memcpy(&val, (u8 *)addr, len); 461 } 462 463 if (!fp) 464 SET_RD(insn, regs, (long)(val.data_ulong << shift) >> shift); 465 else if (len == 8) 466 set_f64_rd(insn, regs, val.data_u64); 467 else 468 set_f32_rd(insn, regs, val.data_ulong); 469 470 regs->epc = epc + INSN_LEN(insn); 471 472 return 0; 473 } 474 475 static int handle_scalar_misaligned_store(struct pt_regs *regs) 476 { 477 union reg_data val; 478 unsigned long epc = regs->epc; 479 unsigned long insn; 480 unsigned long addr = regs->badaddr; 481 int len = 0, fp = 0; 482 483 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr); 484 485 if (!unaligned_enabled) 486 return -1; 487 488 if (user_mode(regs) && (current->thread.align_ctl & PR_UNALIGN_SIGBUS)) 489 return -1; 490 491 if (get_insn(regs, epc, &insn)) 492 return -1; 493 494 regs->epc = 0; 495 496 val.data_ulong = GET_RS2(insn, regs); 497 498 if ((insn & INSN_MASK_SW) == INSN_MATCH_SW) { 499 len = 4; 500 #if defined(CONFIG_64BIT) 501 } else if ((insn & INSN_MASK_SD) == INSN_MATCH_SD) { 502 len = 8; 503 #endif 504 } else if ((insn & INSN_MASK_FSD) == INSN_MATCH_FSD) { 505 fp = 1; 506 len = 8; 507 val.data_u64 = GET_F64_RS2(insn, regs); 508 } else if ((insn & INSN_MASK_FSW) == INSN_MATCH_FSW) { 509 fp = 1; 510 len = 4; 511 val.data_ulong = GET_F32_RS2(insn, regs); 512 } else if ((insn & INSN_MASK_SH) == INSN_MATCH_SH) { 513 len = 2; 514 #if defined(CONFIG_64BIT) 515 } else if ((insn & INSN_MASK_C_SD) == INSN_MATCH_C_SD) { 516 len = 8; 517 val.data_ulong = GET_RS2S(insn, regs); 518 } else if ((insn & INSN_MASK_C_SDSP) == INSN_MATCH_C_SDSP) { 519 len = 8; 520 val.data_ulong = GET_RS2C(insn, regs); 521 #endif 522 } else if ((insn & INSN_MASK_C_SW) == INSN_MATCH_C_SW) { 523 len = 4; 524 val.data_ulong = GET_RS2S(insn, regs); 525 } else if ((insn & INSN_MASK_C_SWSP) == INSN_MATCH_C_SWSP) { 526 len = 4; 527 val.data_ulong = GET_RS2C(insn, regs); 528 } else if ((insn & INSN_MASK_C_FSD) == INSN_MATCH_C_FSD) { 529 fp = 1; 530 len = 8; 531 val.data_u64 = GET_F64_RS2S(insn, regs); 532 } else if ((insn & INSN_MASK_C_FSDSP) == INSN_MATCH_C_FSDSP) { 533 fp = 1; 534 len = 8; 535 val.data_u64 = GET_F64_RS2C(insn, regs); 536 #if !defined(CONFIG_64BIT) 537 } else if ((insn & INSN_MASK_C_FSW) == INSN_MATCH_C_FSW) { 538 fp = 1; 539 len = 4; 540 val.data_ulong = GET_F32_RS2S(insn, regs); 541 } else if ((insn & INSN_MASK_C_FSWSP) == INSN_MATCH_C_FSWSP) { 542 fp = 1; 543 len = 4; 544 val.data_ulong = GET_F32_RS2C(insn, regs); 545 #endif 546 } else if ((insn & INSN_MASK_C_SH) == INSN_MATCH_C_SH) { 547 len = 2; 548 val.data_ulong = GET_RS2S(insn, regs); 549 } else { 550 regs->epc = epc; 551 return -1; 552 } 553 554 if (!IS_ENABLED(CONFIG_FPU) && fp) 555 return -EOPNOTSUPP; 556 557 if (user_mode(regs)) { 558 if (copy_to_user((u8 __user *)addr, &val, len)) 559 return -1; 560 } else { 561 memcpy((u8 *)addr, &val, len); 562 } 563 564 regs->epc = epc + INSN_LEN(insn); 565 566 return 0; 567 } 568 569 int handle_misaligned_load(struct pt_regs *regs) 570 { 571 unsigned long epc = regs->epc; 572 unsigned long insn; 573 574 if (IS_ENABLED(CONFIG_RISCV_VECTOR_MISALIGNED)) { 575 if (get_insn(regs, epc, &insn)) 576 return -1; 577 578 if (insn_is_vector(insn)) 579 return handle_vector_misaligned_load(regs); 580 } 581 582 if (IS_ENABLED(CONFIG_RISCV_SCALAR_MISALIGNED)) 583 return handle_scalar_misaligned_load(regs); 584 585 return -1; 586 } 587 588 int handle_misaligned_store(struct pt_regs *regs) 589 { 590 if (IS_ENABLED(CONFIG_RISCV_SCALAR_MISALIGNED)) 591 return handle_scalar_misaligned_store(regs); 592 593 return -1; 594 } 595 596 #ifdef CONFIG_RISCV_VECTOR_MISALIGNED 597 void check_vector_unaligned_access_emulated(struct work_struct *work __always_unused) 598 { 599 long *mas_ptr = this_cpu_ptr(&vector_misaligned_access); 600 unsigned long tmp_var; 601 602 *mas_ptr = RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN; 603 604 kernel_vector_begin(); 605 /* 606 * In pre-13.0.0 versions of GCC, vector registers cannot appear in 607 * the clobber list. This inline asm clobbers v0, but since we do not 608 * currently build the kernel with V enabled, the v0 clobber arg is not 609 * needed (as the compiler will not emit vector code itself). If the kernel 610 * is changed to build with V enabled, the clobber arg will need to be 611 * added here. 612 */ 613 __asm__ __volatile__ ( 614 ".balign 4\n\t" 615 ".option push\n\t" 616 ".option arch, +zve32x\n\t" 617 " vsetivli zero, 1, e16, m1, ta, ma\n\t" // Vectors of 16b 618 " vle16.v v0, (%[ptr])\n\t" // Load bytes 619 ".option pop\n\t" 620 : : [ptr] "r" ((u8 *)&tmp_var + 1)); 621 kernel_vector_end(); 622 } 623 624 bool __init check_vector_unaligned_access_emulated_all_cpus(void) 625 { 626 int cpu; 627 628 /* 629 * While being documented as very slow, schedule_on_each_cpu() is used since 630 * kernel_vector_begin() expects irqs to be enabled or it will panic() 631 */ 632 schedule_on_each_cpu(check_vector_unaligned_access_emulated); 633 634 for_each_online_cpu(cpu) 635 if (per_cpu(vector_misaligned_access, cpu) 636 == RISCV_HWPROBE_MISALIGNED_VECTOR_UNKNOWN) 637 return false; 638 639 return true; 640 } 641 #else 642 bool __init check_vector_unaligned_access_emulated_all_cpus(void) 643 { 644 return false; 645 } 646 #endif 647 648 static bool all_cpus_unaligned_scalar_access_emulated(void) 649 { 650 int cpu; 651 652 for_each_online_cpu(cpu) 653 if (per_cpu(misaligned_access_speed, cpu) != 654 RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED) 655 return false; 656 657 return true; 658 } 659 660 #ifdef CONFIG_RISCV_SCALAR_MISALIGNED 661 662 static bool unaligned_ctl __read_mostly; 663 664 static void check_unaligned_access_emulated(void *arg __always_unused) 665 { 666 int cpu = smp_processor_id(); 667 long *mas_ptr = per_cpu_ptr(&misaligned_access_speed, cpu); 668 unsigned long tmp_var, tmp_val; 669 670 *mas_ptr = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN; 671 672 __asm__ __volatile__ ( 673 " "REG_L" %[tmp], 1(%[ptr])\n" 674 : [tmp] "=r" (tmp_val) : [ptr] "r" (&tmp_var) : "memory"); 675 } 676 677 static int cpu_online_check_unaligned_access_emulated(unsigned int cpu) 678 { 679 long *mas_ptr = per_cpu_ptr(&misaligned_access_speed, cpu); 680 681 check_unaligned_access_emulated(NULL); 682 683 /* 684 * If unaligned_ctl is already set, this means that we detected that all 685 * CPUS uses emulated misaligned access at boot time. If that changed 686 * when hotplugging the new cpu, this is something we don't handle. 687 */ 688 if (unlikely(unaligned_ctl && (*mas_ptr != RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED))) { 689 pr_crit("CPU misaligned accesses non homogeneous (expected all emulated)\n"); 690 return -EINVAL; 691 } 692 693 return 0; 694 } 695 696 bool __init check_unaligned_access_emulated_all_cpus(void) 697 { 698 /* 699 * We can only support PR_UNALIGN controls if all CPUs have misaligned 700 * accesses emulated since tasks requesting such control can run on any 701 * CPU. 702 */ 703 on_each_cpu(check_unaligned_access_emulated, NULL, 1); 704 705 if (!all_cpus_unaligned_scalar_access_emulated()) 706 return false; 707 708 unaligned_ctl = true; 709 return true; 710 } 711 712 bool unaligned_ctl_available(void) 713 { 714 return unaligned_ctl; 715 } 716 #else 717 bool __init check_unaligned_access_emulated_all_cpus(void) 718 { 719 return false; 720 } 721 static int cpu_online_check_unaligned_access_emulated(unsigned int cpu) 722 { 723 return 0; 724 } 725 #endif 726 727 static bool misaligned_traps_delegated; 728 729 #ifdef CONFIG_RISCV_SBI 730 731 static int cpu_online_sbi_unaligned_setup(unsigned int cpu) 732 { 733 if (sbi_fwft_set(SBI_FWFT_MISALIGNED_EXC_DELEG, 1, 0) && 734 misaligned_traps_delegated) { 735 pr_crit("Misaligned trap delegation non homogeneous (expected delegated)"); 736 return -EINVAL; 737 } 738 739 return 0; 740 } 741 742 void __init unaligned_access_init(void) 743 { 744 int ret; 745 746 ret = sbi_fwft_set_online_cpus(SBI_FWFT_MISALIGNED_EXC_DELEG, 1, 0); 747 if (ret) 748 return; 749 750 misaligned_traps_delegated = true; 751 pr_info("SBI misaligned access exception delegation ok\n"); 752 /* 753 * Note that we don't have to take any specific action here, if 754 * the delegation is successful, then 755 * check_unaligned_access_emulated() will verify that indeed the 756 * platform traps on misaligned accesses. 757 */ 758 } 759 #else 760 void __init unaligned_access_init(void) {} 761 762 static int cpu_online_sbi_unaligned_setup(unsigned int cpu __always_unused) 763 { 764 return 0; 765 } 766 767 #endif 768 769 int cpu_online_unaligned_access_init(unsigned int cpu) 770 { 771 int ret; 772 773 ret = cpu_online_sbi_unaligned_setup(cpu); 774 if (ret) 775 return ret; 776 777 return cpu_online_check_unaligned_access_emulated(cpu); 778 } 779 780 bool misaligned_traps_can_delegate(void) 781 { 782 /* 783 * Either we successfully requested misaligned traps delegation for all 784 * CPUs, or the SBI does not implement the FWFT extension but delegated 785 * the exception by default. 786 */ 787 return misaligned_traps_delegated || 788 all_cpus_unaligned_scalar_access_emulated(); 789 } 790 EXPORT_SYMBOL_GPL(misaligned_traps_can_delegate); 791