1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright 2022-2024 Rivos, Inc
4 */
5
6 #ifndef _ASM_CPUFEATURE_H
7 #define _ASM_CPUFEATURE_H
8
9 #include <linux/bitmap.h>
10 #include <linux/jump_label.h>
11 #include <linux/workqueue.h>
12 #include <linux/kconfig.h>
13 #include <linux/percpu-defs.h>
14 #include <linux/threads.h>
15 #include <asm/hwcap.h>
16 #include <asm/cpufeature-macros.h>
17
18 /*
19 * These are probed via a device_initcall(), via either the SBI or directly
20 * from the corresponding CSRs.
21 */
22 struct riscv_cpuinfo {
23 unsigned long mvendorid;
24 unsigned long marchid;
25 unsigned long mimpid;
26 };
27
28 struct riscv_isainfo {
29 DECLARE_BITMAP(isa, RISCV_ISA_EXT_MAX);
30 };
31
32 DECLARE_PER_CPU(struct riscv_cpuinfo, riscv_cpuinfo);
33
34 extern const struct seq_operations cpuinfo_op;
35
36 /* Per-cpu ISA extensions. */
37 extern struct riscv_isainfo hart_isa[NR_CPUS];
38
39 extern u32 thead_vlenb_of;
40
41 void __init riscv_user_isa_enable(void);
42
43 void riscv_clear_hypervisor_csr(void);
44
45 #define _RISCV_ISA_EXT_DATA(_name, _id, _subset_exts, _subset_exts_size, _validate) { \
46 .name = #_name, \
47 .property = #_name, \
48 .id = _id, \
49 .subset_ext_ids = _subset_exts, \
50 .subset_ext_size = _subset_exts_size, \
51 .validate = _validate \
52 }
53
54 #define __RISCV_ISA_EXT_DATA(_name, _id) _RISCV_ISA_EXT_DATA(_name, _id, NULL, 0, NULL)
55
56 #define __RISCV_ISA_EXT_DATA_VALIDATE(_name, _id, _validate) \
57 _RISCV_ISA_EXT_DATA(_name, _id, NULL, 0, _validate)
58
59 /* Used to declare pure "lasso" extension (Zk for instance) */
60 #define __RISCV_ISA_EXT_BUNDLE(_name, _bundled_exts) \
61 _RISCV_ISA_EXT_DATA(_name, RISCV_ISA_EXT_INVALID, _bundled_exts, \
62 ARRAY_SIZE(_bundled_exts), NULL)
63 #define __RISCV_ISA_EXT_BUNDLE_VALIDATE(_name, _bundled_exts, _validate) \
64 _RISCV_ISA_EXT_DATA(_name, RISCV_ISA_EXT_INVALID, _bundled_exts, \
65 ARRAY_SIZE(_bundled_exts), _validate)
66
67 /* Used to declare extensions that are a superset of other extensions (Zvbb for instance) */
68 #define __RISCV_ISA_EXT_SUPERSET(_name, _id, _sub_exts) \
69 _RISCV_ISA_EXT_DATA(_name, _id, _sub_exts, ARRAY_SIZE(_sub_exts), NULL)
70 #define __RISCV_ISA_EXT_SUPERSET_VALIDATE(_name, _id, _sub_exts, _validate) \
71 _RISCV_ISA_EXT_DATA(_name, _id, _sub_exts, ARRAY_SIZE(_sub_exts), _validate)
72
73 bool __init check_unaligned_access_emulated_all_cpus(void);
74 void unaligned_access_init(void);
75 int cpu_online_unaligned_access_init(unsigned int cpu);
76 #if defined(CONFIG_RISCV_SCALAR_MISALIGNED)
77 void unaligned_emulation_finish(void);
78 bool unaligned_ctl_available(void);
79 #else
unaligned_ctl_available(void)80 static inline bool unaligned_ctl_available(void)
81 {
82 return false;
83 }
84 #endif
85
86 #if defined(CONFIG_RISCV_MISALIGNED)
87 DECLARE_PER_CPU(long, misaligned_access_speed);
88 bool misaligned_traps_can_delegate(void);
89 #else
misaligned_traps_can_delegate(void)90 static inline bool misaligned_traps_can_delegate(void)
91 {
92 return false;
93 }
94 #endif
95
96 bool __init check_vector_unaligned_access_emulated_all_cpus(void);
97 #if defined(CONFIG_RISCV_VECTOR_MISALIGNED)
98 void check_vector_unaligned_access_emulated(struct work_struct *work __always_unused);
99 DECLARE_PER_CPU(long, vector_misaligned_access);
100 #endif
101
102 #if defined(CONFIG_RISCV_PROBE_UNALIGNED_ACCESS)
103 DECLARE_STATIC_KEY_FALSE(fast_unaligned_access_speed_key);
104
has_fast_unaligned_accesses(void)105 static __always_inline bool has_fast_unaligned_accesses(void)
106 {
107 return static_branch_likely(&fast_unaligned_access_speed_key);
108 }
109 #else
has_fast_unaligned_accesses(void)110 static __always_inline bool has_fast_unaligned_accesses(void)
111 {
112 if (IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
113 return true;
114 else
115 return false;
116 }
117 #endif
118
119 unsigned long riscv_get_elf_hwcap(void);
120
121 struct riscv_isa_ext_data {
122 const unsigned int id;
123 const char *name;
124 const char *property;
125 const unsigned int *subset_ext_ids;
126 const unsigned int subset_ext_size;
127 int (*validate)(const struct riscv_isa_ext_data *data, const unsigned long *isa_bitmap);
128 };
129
130 extern const struct riscv_isa_ext_data riscv_isa_ext[];
131 extern const size_t riscv_isa_ext_count;
132 extern bool riscv_isa_fallback;
133
134 unsigned long riscv_isa_extension_base(const unsigned long *isa_bitmap);
riscv_cpu_has_extension_likely(int cpu,const unsigned long ext)135 static __always_inline bool riscv_cpu_has_extension_likely(int cpu, const unsigned long ext)
136 {
137 compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX");
138
139 if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE) &&
140 __riscv_has_extension_likely(STANDARD_EXT, ext))
141 return true;
142
143 return __riscv_isa_extension_available(hart_isa[cpu].isa, ext);
144 }
145
riscv_cpu_has_extension_unlikely(int cpu,const unsigned long ext)146 static __always_inline bool riscv_cpu_has_extension_unlikely(int cpu, const unsigned long ext)
147 {
148 compiletime_assert(ext < RISCV_ISA_EXT_MAX, "ext must be < RISCV_ISA_EXT_MAX");
149
150 if (IS_ENABLED(CONFIG_RISCV_ALTERNATIVE) &&
151 __riscv_has_extension_unlikely(STANDARD_EXT, ext))
152 return true;
153
154 return __riscv_isa_extension_available(hart_isa[cpu].isa, ext);
155 }
156
cpu_supports_shadow_stack(void)157 static inline bool cpu_supports_shadow_stack(void)
158 {
159 return (IS_ENABLED(CONFIG_RISCV_USER_CFI) &&
160 riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICFISS));
161 }
162
cpu_supports_indirect_br_lp_instr(void)163 static inline bool cpu_supports_indirect_br_lp_instr(void)
164 {
165 return (IS_ENABLED(CONFIG_RISCV_USER_CFI) &&
166 riscv_has_extension_unlikely(RISCV_ISA_EXT_ZICFILP));
167 }
168
169 #endif
170