1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright 2022-2024 Rivos, Inc 4 */ 5 6 #ifndef _ASM_CPUFEATURE_H 7 #define _ASM_CPUFEATURE_H 8 9 #include <linux/bitmap.h> 10 #include <linux/jump_label.h> 11 #include <linux/workqueue.h> 12 #include <asm/hwcap.h> 13 #include <asm/alternative-macros.h> 14 #include <asm/errno.h> 15 16 /* 17 * These are probed via a device_initcall(), via either the SBI or directly 18 * from the corresponding CSRs. 19 */ 20 struct riscv_cpuinfo { 21 unsigned long mvendorid; 22 unsigned long marchid; 23 unsigned long mimpid; 24 }; 25 26 struct riscv_isainfo { 27 DECLARE_BITMAP(isa, RISCV_ISA_EXT_MAX); 28 }; 29 30 DECLARE_PER_CPU(struct riscv_cpuinfo, riscv_cpuinfo); 31 32 /* Per-cpu ISA extensions. */ 33 extern struct riscv_isainfo hart_isa[NR_CPUS]; 34 35 void riscv_user_isa_enable(void); 36 37 #define _RISCV_ISA_EXT_DATA(_name, _id, _subset_exts, _subset_exts_size, _validate) { \ 38 .name = #_name, \ 39 .property = #_name, \ 40 .id = _id, \ 41 .subset_ext_ids = _subset_exts, \ 42 .subset_ext_size = _subset_exts_size, \ 43 .validate = _validate \ 44 } 45 46 #define __RISCV_ISA_EXT_DATA(_name, _id) _RISCV_ISA_EXT_DATA(_name, _id, NULL, 0, NULL) 47 48 #define __RISCV_ISA_EXT_DATA_VALIDATE(_name, _id, _validate) \ 49 _RISCV_ISA_EXT_DATA(_name, _id, NULL, 0, _validate) 50 51 /* Used to declare pure "lasso" extension (Zk for instance) */ 52 #define __RISCV_ISA_EXT_BUNDLE(_name, _bundled_exts) \ 53 _RISCV_ISA_EXT_DATA(_name, RISCV_ISA_EXT_INVALID, _bundled_exts, \ 54 ARRAY_SIZE(_bundled_exts), NULL) 55 56 /* Used to declare extensions that are a superset of other extensions (Zvbb for instance) */ 57 #define __RISCV_ISA_EXT_SUPERSET(_name, _id, _sub_exts) \ 58 _RISCV_ISA_EXT_DATA(_name, _id, _sub_exts, ARRAY_SIZE(_sub_exts), NULL) 59 #define __RISCV_ISA_EXT_SUPERSET_VALIDATE(_name, _id, _sub_exts, _validate) \ 60 _RISCV_ISA_EXT_DATA(_name, _id, _sub_exts, ARRAY_SIZE(_sub_exts), _validate) 61 62 #if defined(CONFIG_RISCV_SCALAR_MISALIGNED) 63 bool check_unaligned_access_emulated_all_cpus(void); 64 void check_unaligned_access_emulated(struct work_struct *work __always_unused); 65 void unaligned_emulation_finish(void); 66 bool unaligned_ctl_available(void); 67 DECLARE_PER_CPU(long, misaligned_access_speed); 68 #else 69 static inline bool unaligned_ctl_available(void) 70 { 71 return false; 72 } 73 #endif 74 75 #if defined(CONFIG_RISCV_PROBE_UNALIGNED_ACCESS) 76 DECLARE_STATIC_KEY_FALSE(fast_unaligned_access_speed_key); 77 78 static __always_inline bool has_fast_unaligned_accesses(void) 79 { 80 return static_branch_likely(&fast_unaligned_access_speed_key); 81 } 82 #else 83 static __always_inline bool has_fast_unaligned_accesses(void) 84 { 85 if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) 86 return true; 87 else 88 return false; 89 } 90 #endif 91 92 unsigned long riscv_get_elf_hwcap(void); 93 94 struct riscv_isa_ext_data { 95 const unsigned int id; 96 const char *name; 97 const char *property; 98 const unsigned int *subset_ext_ids; 99 const unsigned int subset_ext_size; 100 int (*validate)(const struct riscv_isa_ext_data *data, const unsigned long *isa_bitmap); 101 }; 102 103 extern const struct riscv_isa_ext_data riscv_isa_ext[]; 104 extern const size_t riscv_isa_ext_count; 105 extern bool riscv_isa_fallback; 106 107 unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap); 108 109 #define STANDARD_EXT 0 110 111 bool __riscv_isa_extension_available(const unsigned long *isa_bitmap, unsigned int bit); 112 #define riscv_isa_extension_available(isa_bitmap, ext) \ 113 __riscv_isa_extension_available(isa_bitmap, RISCV_ISA_EXT_##ext) 114 115 static __always_inline bool __riscv_has_extension_likely(const unsigned long vendor, 116 const unsigned long ext) 117 { 118 asm goto(ALTERNATIVE("j %l[l_no]", "nop", %[vendor], %[ext], 1) 119 : 120 : [vendor] "i" (vendor), [ext] "i" (ext) 121 : 122 : l_no); 123 124 return true; 125 l_no: 126 return false; 127 } 128 129 static __always_inline bool __riscv_has_extension_unlikely(const unsigned long vendor, 130 const unsigned long ext) 131 { 132 asm goto(ALTERNATIVE("nop", "j %l[l_yes]", %[vendor], %[ext], 1) 133 : 134 : [vendor] "i" (vendor), [ext] "i" (ext) 135 : 136 : l_yes); 137 138 return false; 139 l_yes: 140 return true; 141 } 142 143 static __always_inline bool riscv_has_extension_unlikely(const unsigned long ext) 144 { 145 compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX"); 146 147 if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) 148 return __riscv_has_extension_unlikely(STANDARD_EXT, ext); 149 150 return __riscv_isa_extension_available(NULL, ext); 151 } 152 153 static __always_inline bool riscv_has_extension_likely(const unsigned long ext) 154 { 155 compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX"); 156 157 if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE)) 158 return __riscv_has_extension_likely(STANDARD_EXT, ext); 159 160 return __riscv_isa_extension_available(NULL, ext); 161 } 162 163 static __always_inline bool riscv_cpu_has_extension_likely(int cpu, const unsigned long ext) 164 { 165 compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX"); 166 167 if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE) && 168 __riscv_has_extension_likely(STANDARD_EXT, ext)) 169 return true; 170 171 return __riscv_isa_extension_available(hart_isa[cpu].isa, ext); 172 } 173 174 static __always_inline bool riscv_cpu_has_extension_unlikely(int cpu, const unsigned long ext) 175 { 176 compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX"); 177 178 if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE) && 179 __riscv_has_extension_unlikely(STANDARD_EXT, ext)) 180 return true; 181 182 return __riscv_isa_extension_available(hart_isa[cpu].isa, ext); 183 } 184 185 #endif 186