1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2020 Western Digital Corporation or its affiliates. 4 */ 5 #include <linux/kernel.h> 6 #include <linux/init.h> 7 #include <linux/mm.h> 8 #include <linux/module.h> 9 #include <linux/perf_event.h> 10 #include <linux/irq.h> 11 #include <linux/stringify.h> 12 13 #include <asm/processor.h> 14 #include <asm/ptrace.h> 15 #include <asm/csr.h> 16 #include <asm/entry-common.h> 17 #include <asm/hwprobe.h> 18 #include <asm/cpufeature.h> 19 20 #define INSN_MATCH_LB 0x3 21 #define INSN_MASK_LB 0x707f 22 #define INSN_MATCH_LH 0x1003 23 #define INSN_MASK_LH 0x707f 24 #define INSN_MATCH_LW 0x2003 25 #define INSN_MASK_LW 0x707f 26 #define INSN_MATCH_LD 0x3003 27 #define INSN_MASK_LD 0x707f 28 #define INSN_MATCH_LBU 0x4003 29 #define INSN_MASK_LBU 0x707f 30 #define INSN_MATCH_LHU 0x5003 31 #define INSN_MASK_LHU 0x707f 32 #define INSN_MATCH_LWU 0x6003 33 #define INSN_MASK_LWU 0x707f 34 #define INSN_MATCH_SB 0x23 35 #define INSN_MASK_SB 0x707f 36 #define INSN_MATCH_SH 0x1023 37 #define INSN_MASK_SH 0x707f 38 #define INSN_MATCH_SW 0x2023 39 #define INSN_MASK_SW 0x707f 40 #define INSN_MATCH_SD 0x3023 41 #define INSN_MASK_SD 0x707f 42 43 #define INSN_MATCH_FLW 0x2007 44 #define INSN_MASK_FLW 0x707f 45 #define INSN_MATCH_FLD 0x3007 46 #define INSN_MASK_FLD 0x707f 47 #define INSN_MATCH_FLQ 0x4007 48 #define INSN_MASK_FLQ 0x707f 49 #define INSN_MATCH_FSW 0x2027 50 #define INSN_MASK_FSW 0x707f 51 #define INSN_MATCH_FSD 0x3027 52 #define INSN_MASK_FSD 0x707f 53 #define INSN_MATCH_FSQ 0x4027 54 #define INSN_MASK_FSQ 0x707f 55 56 #define INSN_MATCH_C_LD 0x6000 57 #define INSN_MASK_C_LD 0xe003 58 #define INSN_MATCH_C_SD 0xe000 59 #define INSN_MASK_C_SD 0xe003 60 #define INSN_MATCH_C_LW 0x4000 61 #define INSN_MASK_C_LW 0xe003 62 #define INSN_MATCH_C_SW 0xc000 63 #define INSN_MASK_C_SW 0xe003 64 #define INSN_MATCH_C_LDSP 0x6002 65 #define INSN_MASK_C_LDSP 0xe003 66 #define INSN_MATCH_C_SDSP 0xe002 67 #define INSN_MASK_C_SDSP 0xe003 68 #define INSN_MATCH_C_LWSP 0x4002 69 #define INSN_MASK_C_LWSP 0xe003 70 #define INSN_MATCH_C_SWSP 0xc002 71 #define INSN_MASK_C_SWSP 0xe003 72 73 #define INSN_MATCH_C_FLD 0x2000 74 #define INSN_MASK_C_FLD 0xe003 75 #define INSN_MATCH_C_FLW 0x6000 76 #define INSN_MASK_C_FLW 0xe003 77 #define INSN_MATCH_C_FSD 0xa000 78 #define INSN_MASK_C_FSD 0xe003 79 #define INSN_MATCH_C_FSW 0xe000 80 #define INSN_MASK_C_FSW 0xe003 81 #define INSN_MATCH_C_FLDSP 0x2002 82 #define INSN_MASK_C_FLDSP 0xe003 83 #define INSN_MATCH_C_FSDSP 0xa002 84 #define INSN_MASK_C_FSDSP 0xe003 85 #define INSN_MATCH_C_FLWSP 0x6002 86 #define INSN_MASK_C_FLWSP 0xe003 87 #define INSN_MATCH_C_FSWSP 0xe002 88 #define INSN_MASK_C_FSWSP 0xe003 89 90 #define INSN_LEN(insn) ((((insn) & 0x3) < 0x3) ? 2 : 4) 91 92 #if defined(CONFIG_64BIT) 93 #define LOG_REGBYTES 3 94 #define XLEN 64 95 #else 96 #define LOG_REGBYTES 2 97 #define XLEN 32 98 #endif 99 #define REGBYTES (1 << LOG_REGBYTES) 100 #define XLEN_MINUS_16 ((XLEN) - 16) 101 102 #define SH_RD 7 103 #define SH_RS1 15 104 #define SH_RS2 20 105 #define SH_RS2C 2 106 107 #define RV_X(x, s, n) (((x) >> (s)) & ((1 << (n)) - 1)) 108 #define RVC_LW_IMM(x) ((RV_X(x, 6, 1) << 2) | \ 109 (RV_X(x, 10, 3) << 3) | \ 110 (RV_X(x, 5, 1) << 6)) 111 #define RVC_LD_IMM(x) ((RV_X(x, 10, 3) << 3) | \ 112 (RV_X(x, 5, 2) << 6)) 113 #define RVC_LWSP_IMM(x) ((RV_X(x, 4, 3) << 2) | \ 114 (RV_X(x, 12, 1) << 5) | \ 115 (RV_X(x, 2, 2) << 6)) 116 #define RVC_LDSP_IMM(x) ((RV_X(x, 5, 2) << 3) | \ 117 (RV_X(x, 12, 1) << 5) | \ 118 (RV_X(x, 2, 3) << 6)) 119 #define RVC_SWSP_IMM(x) ((RV_X(x, 9, 4) << 2) | \ 120 (RV_X(x, 7, 2) << 6)) 121 #define RVC_SDSP_IMM(x) ((RV_X(x, 10, 3) << 3) | \ 122 (RV_X(x, 7, 3) << 6)) 123 #define RVC_RS1S(insn) (8 + RV_X(insn, SH_RD, 3)) 124 #define RVC_RS2S(insn) (8 + RV_X(insn, SH_RS2C, 3)) 125 #define RVC_RS2(insn) RV_X(insn, SH_RS2C, 5) 126 127 #define SHIFT_RIGHT(x, y) \ 128 ((y) < 0 ? ((x) << -(y)) : ((x) >> (y))) 129 130 #define REG_MASK \ 131 ((1 << (5 + LOG_REGBYTES)) - (1 << LOG_REGBYTES)) 132 133 #define REG_OFFSET(insn, pos) \ 134 (SHIFT_RIGHT((insn), (pos) - LOG_REGBYTES) & REG_MASK) 135 136 #define REG_PTR(insn, pos, regs) \ 137 (ulong *)((ulong)(regs) + REG_OFFSET(insn, pos)) 138 139 #define GET_RS1(insn, regs) (*REG_PTR(insn, SH_RS1, regs)) 140 #define GET_RS2(insn, regs) (*REG_PTR(insn, SH_RS2, regs)) 141 #define GET_RS1S(insn, regs) (*REG_PTR(RVC_RS1S(insn), 0, regs)) 142 #define GET_RS2S(insn, regs) (*REG_PTR(RVC_RS2S(insn), 0, regs)) 143 #define GET_RS2C(insn, regs) (*REG_PTR(insn, SH_RS2C, regs)) 144 #define GET_SP(regs) (*REG_PTR(2, 0, regs)) 145 #define SET_RD(insn, regs, val) (*REG_PTR(insn, SH_RD, regs) = (val)) 146 #define IMM_I(insn) ((s32)(insn) >> 20) 147 #define IMM_S(insn) (((s32)(insn) >> 25 << 5) | \ 148 (s32)(((insn) >> 7) & 0x1f)) 149 #define MASK_FUNCT3 0x7000 150 151 #define GET_PRECISION(insn) (((insn) >> 25) & 3) 152 #define GET_RM(insn) (((insn) >> 12) & 7) 153 #define PRECISION_S 0 154 #define PRECISION_D 1 155 156 #ifdef CONFIG_FPU 157 158 #define FP_GET_RD(insn) (insn >> 7 & 0x1F) 159 160 extern void put_f32_reg(unsigned long fp_reg, unsigned long value); 161 162 static int set_f32_rd(unsigned long insn, struct pt_regs *regs, 163 unsigned long val) 164 { 165 unsigned long fp_reg = FP_GET_RD(insn); 166 167 put_f32_reg(fp_reg, val); 168 regs->status |= SR_FS_DIRTY; 169 170 return 0; 171 } 172 173 extern void put_f64_reg(unsigned long fp_reg, unsigned long value); 174 175 static int set_f64_rd(unsigned long insn, struct pt_regs *regs, u64 val) 176 { 177 unsigned long fp_reg = FP_GET_RD(insn); 178 unsigned long value; 179 180 #if __riscv_xlen == 32 181 value = (unsigned long) &val; 182 #else 183 value = val; 184 #endif 185 put_f64_reg(fp_reg, value); 186 regs->status |= SR_FS_DIRTY; 187 188 return 0; 189 } 190 191 #if __riscv_xlen == 32 192 extern void get_f64_reg(unsigned long fp_reg, u64 *value); 193 194 static u64 get_f64_rs(unsigned long insn, u8 fp_reg_offset, 195 struct pt_regs *regs) 196 { 197 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F; 198 u64 val; 199 200 get_f64_reg(fp_reg, &val); 201 regs->status |= SR_FS_DIRTY; 202 203 return val; 204 } 205 #else 206 207 extern unsigned long get_f64_reg(unsigned long fp_reg); 208 209 static unsigned long get_f64_rs(unsigned long insn, u8 fp_reg_offset, 210 struct pt_regs *regs) 211 { 212 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F; 213 unsigned long val; 214 215 val = get_f64_reg(fp_reg); 216 regs->status |= SR_FS_DIRTY; 217 218 return val; 219 } 220 221 #endif 222 223 extern unsigned long get_f32_reg(unsigned long fp_reg); 224 225 static unsigned long get_f32_rs(unsigned long insn, u8 fp_reg_offset, 226 struct pt_regs *regs) 227 { 228 unsigned long fp_reg = (insn >> fp_reg_offset) & 0x1F; 229 unsigned long val; 230 231 val = get_f32_reg(fp_reg); 232 regs->status |= SR_FS_DIRTY; 233 234 return val; 235 } 236 237 #else /* CONFIG_FPU */ 238 static void set_f32_rd(unsigned long insn, struct pt_regs *regs, 239 unsigned long val) {} 240 241 static void set_f64_rd(unsigned long insn, struct pt_regs *regs, u64 val) {} 242 243 static unsigned long get_f64_rs(unsigned long insn, u8 fp_reg_offset, 244 struct pt_regs *regs) 245 { 246 return 0; 247 } 248 249 static unsigned long get_f32_rs(unsigned long insn, u8 fp_reg_offset, 250 struct pt_regs *regs) 251 { 252 return 0; 253 } 254 255 #endif 256 257 #define GET_F64_RS2(insn, regs) (get_f64_rs(insn, 20, regs)) 258 #define GET_F64_RS2C(insn, regs) (get_f64_rs(insn, 2, regs)) 259 #define GET_F64_RS2S(insn, regs) (get_f64_rs(RVC_RS2S(insn), 0, regs)) 260 261 #define GET_F32_RS2(insn, regs) (get_f32_rs(insn, 20, regs)) 262 #define GET_F32_RS2C(insn, regs) (get_f32_rs(insn, 2, regs)) 263 #define GET_F32_RS2S(insn, regs) (get_f32_rs(RVC_RS2S(insn), 0, regs)) 264 265 #define __read_insn(regs, insn, insn_addr, type) \ 266 ({ \ 267 int __ret; \ 268 \ 269 if (user_mode(regs)) { \ 270 __ret = __get_user(insn, (type __user *) insn_addr); \ 271 } else { \ 272 insn = *(type *)insn_addr; \ 273 __ret = 0; \ 274 } \ 275 \ 276 __ret; \ 277 }) 278 279 static inline int get_insn(struct pt_regs *regs, ulong epc, ulong *r_insn) 280 { 281 ulong insn = 0; 282 283 if (epc & 0x2) { 284 ulong tmp = 0; 285 286 if (__read_insn(regs, insn, epc, u16)) 287 return -EFAULT; 288 /* __get_user() uses regular "lw" which sign extend the loaded 289 * value make sure to clear higher order bits in case we "or" it 290 * below with the upper 16 bits half. 291 */ 292 insn &= GENMASK(15, 0); 293 if ((insn & __INSN_LENGTH_MASK) != __INSN_LENGTH_32) { 294 *r_insn = insn; 295 return 0; 296 } 297 epc += sizeof(u16); 298 if (__read_insn(regs, tmp, epc, u16)) 299 return -EFAULT; 300 *r_insn = (tmp << 16) | insn; 301 302 return 0; 303 } else { 304 if (__read_insn(regs, insn, epc, u32)) 305 return -EFAULT; 306 if ((insn & __INSN_LENGTH_MASK) == __INSN_LENGTH_32) { 307 *r_insn = insn; 308 return 0; 309 } 310 insn &= GENMASK(15, 0); 311 *r_insn = insn; 312 313 return 0; 314 } 315 } 316 317 union reg_data { 318 u8 data_bytes[8]; 319 ulong data_ulong; 320 u64 data_u64; 321 }; 322 323 static bool unaligned_ctl __read_mostly; 324 325 /* sysctl hooks */ 326 int unaligned_enabled __read_mostly = 1; /* Enabled by default */ 327 328 int handle_misaligned_load(struct pt_regs *regs) 329 { 330 union reg_data val; 331 unsigned long epc = regs->epc; 332 unsigned long insn; 333 unsigned long addr = regs->badaddr; 334 int fp = 0, shift = 0, len = 0; 335 336 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr); 337 338 #ifdef CONFIG_RISCV_PROBE_UNALIGNED_ACCESS 339 *this_cpu_ptr(&misaligned_access_speed) = RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED; 340 #endif 341 342 if (!unaligned_enabled) 343 return -1; 344 345 if (user_mode(regs) && (current->thread.align_ctl & PR_UNALIGN_SIGBUS)) 346 return -1; 347 348 if (get_insn(regs, epc, &insn)) 349 return -1; 350 351 regs->epc = 0; 352 353 if ((insn & INSN_MASK_LW) == INSN_MATCH_LW) { 354 len = 4; 355 shift = 8 * (sizeof(unsigned long) - len); 356 #if defined(CONFIG_64BIT) 357 } else if ((insn & INSN_MASK_LD) == INSN_MATCH_LD) { 358 len = 8; 359 shift = 8 * (sizeof(unsigned long) - len); 360 } else if ((insn & INSN_MASK_LWU) == INSN_MATCH_LWU) { 361 len = 4; 362 #endif 363 } else if ((insn & INSN_MASK_FLD) == INSN_MATCH_FLD) { 364 fp = 1; 365 len = 8; 366 } else if ((insn & INSN_MASK_FLW) == INSN_MATCH_FLW) { 367 fp = 1; 368 len = 4; 369 } else if ((insn & INSN_MASK_LH) == INSN_MATCH_LH) { 370 len = 2; 371 shift = 8 * (sizeof(unsigned long) - len); 372 } else if ((insn & INSN_MASK_LHU) == INSN_MATCH_LHU) { 373 len = 2; 374 #if defined(CONFIG_64BIT) 375 } else if ((insn & INSN_MASK_C_LD) == INSN_MATCH_C_LD) { 376 len = 8; 377 shift = 8 * (sizeof(unsigned long) - len); 378 insn = RVC_RS2S(insn) << SH_RD; 379 } else if ((insn & INSN_MASK_C_LDSP) == INSN_MATCH_C_LDSP && 380 ((insn >> SH_RD) & 0x1f)) { 381 len = 8; 382 shift = 8 * (sizeof(unsigned long) - len); 383 #endif 384 } else if ((insn & INSN_MASK_C_LW) == INSN_MATCH_C_LW) { 385 len = 4; 386 shift = 8 * (sizeof(unsigned long) - len); 387 insn = RVC_RS2S(insn) << SH_RD; 388 } else if ((insn & INSN_MASK_C_LWSP) == INSN_MATCH_C_LWSP && 389 ((insn >> SH_RD) & 0x1f)) { 390 len = 4; 391 shift = 8 * (sizeof(unsigned long) - len); 392 } else if ((insn & INSN_MASK_C_FLD) == INSN_MATCH_C_FLD) { 393 fp = 1; 394 len = 8; 395 insn = RVC_RS2S(insn) << SH_RD; 396 } else if ((insn & INSN_MASK_C_FLDSP) == INSN_MATCH_C_FLDSP) { 397 fp = 1; 398 len = 8; 399 #if defined(CONFIG_32BIT) 400 } else if ((insn & INSN_MASK_C_FLW) == INSN_MATCH_C_FLW) { 401 fp = 1; 402 len = 4; 403 insn = RVC_RS2S(insn) << SH_RD; 404 } else if ((insn & INSN_MASK_C_FLWSP) == INSN_MATCH_C_FLWSP) { 405 fp = 1; 406 len = 4; 407 #endif 408 } else { 409 regs->epc = epc; 410 return -1; 411 } 412 413 if (!IS_ENABLED(CONFIG_FPU) && fp) 414 return -EOPNOTSUPP; 415 416 val.data_u64 = 0; 417 if (user_mode(regs)) { 418 if (copy_from_user(&val, (u8 __user *)addr, len)) 419 return -1; 420 } else { 421 memcpy(&val, (u8 *)addr, len); 422 } 423 424 if (!fp) 425 SET_RD(insn, regs, val.data_ulong << shift >> shift); 426 else if (len == 8) 427 set_f64_rd(insn, regs, val.data_u64); 428 else 429 set_f32_rd(insn, regs, val.data_ulong); 430 431 regs->epc = epc + INSN_LEN(insn); 432 433 return 0; 434 } 435 436 int handle_misaligned_store(struct pt_regs *regs) 437 { 438 union reg_data val; 439 unsigned long epc = regs->epc; 440 unsigned long insn; 441 unsigned long addr = regs->badaddr; 442 int len = 0, fp = 0; 443 444 perf_sw_event(PERF_COUNT_SW_ALIGNMENT_FAULTS, 1, regs, addr); 445 446 if (!unaligned_enabled) 447 return -1; 448 449 if (user_mode(regs) && (current->thread.align_ctl & PR_UNALIGN_SIGBUS)) 450 return -1; 451 452 if (get_insn(regs, epc, &insn)) 453 return -1; 454 455 regs->epc = 0; 456 457 val.data_ulong = GET_RS2(insn, regs); 458 459 if ((insn & INSN_MASK_SW) == INSN_MATCH_SW) { 460 len = 4; 461 #if defined(CONFIG_64BIT) 462 } else if ((insn & INSN_MASK_SD) == INSN_MATCH_SD) { 463 len = 8; 464 #endif 465 } else if ((insn & INSN_MASK_FSD) == INSN_MATCH_FSD) { 466 fp = 1; 467 len = 8; 468 val.data_u64 = GET_F64_RS2(insn, regs); 469 } else if ((insn & INSN_MASK_FSW) == INSN_MATCH_FSW) { 470 fp = 1; 471 len = 4; 472 val.data_ulong = GET_F32_RS2(insn, regs); 473 } else if ((insn & INSN_MASK_SH) == INSN_MATCH_SH) { 474 len = 2; 475 #if defined(CONFIG_64BIT) 476 } else if ((insn & INSN_MASK_C_SD) == INSN_MATCH_C_SD) { 477 len = 8; 478 val.data_ulong = GET_RS2S(insn, regs); 479 } else if ((insn & INSN_MASK_C_SDSP) == INSN_MATCH_C_SDSP) { 480 len = 8; 481 val.data_ulong = GET_RS2C(insn, regs); 482 #endif 483 } else if ((insn & INSN_MASK_C_SW) == INSN_MATCH_C_SW) { 484 len = 4; 485 val.data_ulong = GET_RS2S(insn, regs); 486 } else if ((insn & INSN_MASK_C_SWSP) == INSN_MATCH_C_SWSP) { 487 len = 4; 488 val.data_ulong = GET_RS2C(insn, regs); 489 } else if ((insn & INSN_MASK_C_FSD) == INSN_MATCH_C_FSD) { 490 fp = 1; 491 len = 8; 492 val.data_u64 = GET_F64_RS2S(insn, regs); 493 } else if ((insn & INSN_MASK_C_FSDSP) == INSN_MATCH_C_FSDSP) { 494 fp = 1; 495 len = 8; 496 val.data_u64 = GET_F64_RS2C(insn, regs); 497 #if !defined(CONFIG_64BIT) 498 } else if ((insn & INSN_MASK_C_FSW) == INSN_MATCH_C_FSW) { 499 fp = 1; 500 len = 4; 501 val.data_ulong = GET_F32_RS2S(insn, regs); 502 } else if ((insn & INSN_MASK_C_FSWSP) == INSN_MATCH_C_FSWSP) { 503 fp = 1; 504 len = 4; 505 val.data_ulong = GET_F32_RS2C(insn, regs); 506 #endif 507 } else { 508 regs->epc = epc; 509 return -1; 510 } 511 512 if (!IS_ENABLED(CONFIG_FPU) && fp) 513 return -EOPNOTSUPP; 514 515 if (user_mode(regs)) { 516 if (copy_to_user((u8 __user *)addr, &val, len)) 517 return -1; 518 } else { 519 memcpy((u8 *)addr, &val, len); 520 } 521 522 regs->epc = epc + INSN_LEN(insn); 523 524 return 0; 525 } 526 527 static bool check_unaligned_access_emulated(int cpu) 528 { 529 long *mas_ptr = per_cpu_ptr(&misaligned_access_speed, cpu); 530 unsigned long tmp_var, tmp_val; 531 bool misaligned_emu_detected; 532 533 *mas_ptr = RISCV_HWPROBE_MISALIGNED_SCALAR_UNKNOWN; 534 535 __asm__ __volatile__ ( 536 " "REG_L" %[tmp], 1(%[ptr])\n" 537 : [tmp] "=r" (tmp_val) : [ptr] "r" (&tmp_var) : "memory"); 538 539 misaligned_emu_detected = (*mas_ptr == RISCV_HWPROBE_MISALIGNED_SCALAR_EMULATED); 540 /* 541 * If unaligned_ctl is already set, this means that we detected that all 542 * CPUS uses emulated misaligned access at boot time. If that changed 543 * when hotplugging the new cpu, this is something we don't handle. 544 */ 545 if (unlikely(unaligned_ctl && !misaligned_emu_detected)) { 546 pr_crit("CPU misaligned accesses non homogeneous (expected all emulated)\n"); 547 while (true) 548 cpu_relax(); 549 } 550 551 return misaligned_emu_detected; 552 } 553 554 bool check_unaligned_access_emulated_all_cpus(void) 555 { 556 int cpu; 557 558 /* 559 * We can only support PR_UNALIGN controls if all CPUs have misaligned 560 * accesses emulated since tasks requesting such control can run on any 561 * CPU. 562 */ 563 for_each_online_cpu(cpu) 564 if (!check_unaligned_access_emulated(cpu)) 565 return false; 566 567 unaligned_ctl = true; 568 return true; 569 } 570 571 bool unaligned_ctl_available(void) 572 { 573 return unaligned_ctl; 574 } 575