xref: /linux/arch/arm64/kernel/cpufeature.c (revision c8bc81a52d5a2ac2e4b257ae123677cf94112755)
1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Contains CPU feature definitions
4  *
5  * Copyright (C) 2015 ARM Ltd.
6  *
7  * A note for the weary kernel hacker: the code here is confusing and hard to
8  * follow! That's partly because it's solving a nasty problem, but also because
9  * there's a little bit of over-abstraction that tends to obscure what's going
10  * on behind a maze of helper functions and macros.
11  *
12  * The basic problem is that hardware folks have started gluing together CPUs
13  * with distinct architectural features; in some cases even creating SoCs where
14  * user-visible instructions are available only on a subset of the available
15  * cores. We try to address this by snapshotting the feature registers of the
16  * boot CPU and comparing these with the feature registers of each secondary
17  * CPU when bringing them up. If there is a mismatch, then we update the
18  * snapshot state to indicate the lowest-common denominator of the feature,
19  * known as the "safe" value. This snapshot state can be queried to view the
20  * "sanitised" value of a feature register.
21  *
22  * The sanitised register values are used to decide which capabilities we
23  * have in the system. These may be in the form of traditional "hwcaps"
24  * advertised to userspace or internal "cpucaps" which are used to configure
25  * things like alternative patching and static keys. While a feature mismatch
26  * may result in a TAINT_CPU_OUT_OF_SPEC kernel taint, a capability mismatch
27  * may prevent a CPU from being onlined at all.
28  *
29  * Some implementation details worth remembering:
30  *
31  * - Mismatched features are *always* sanitised to a "safe" value, which
32  *   usually indicates that the feature is not supported.
33  *
34  * - A mismatched feature marked with FTR_STRICT will cause a "SANITY CHECK"
35  *   warning when onlining an offending CPU and the kernel will be tainted
36  *   with TAINT_CPU_OUT_OF_SPEC.
37  *
38  * - Features marked as FTR_VISIBLE have their sanitised value visible to
39  *   userspace. FTR_VISIBLE features in registers that are only visible
40  *   to EL0 by trapping *must* have a corresponding HWCAP so that late
41  *   onlining of CPUs cannot lead to features disappearing at runtime.
42  *
43  * - A "feature" is typically a 4-bit register field. A "capability" is the
44  *   high-level description derived from the sanitised field value.
45  *
46  * - Read the Arm ARM (DDI 0487F.a) section D13.1.3 ("Principles of the ID
47  *   scheme for fields in ID registers") to understand when feature fields
48  *   may be signed or unsigned (FTR_SIGNED and FTR_UNSIGNED accordingly).
49  *
50  * - KVM exposes its own view of the feature registers to guest operating
51  *   systems regardless of FTR_VISIBLE. This is typically driven from the
52  *   sanitised register values to allow virtual CPUs to be migrated between
53  *   arbitrary physical CPUs, but some features not present on the host are
54  *   also advertised and emulated. Look at sys_reg_descs[] for the gory
55  *   details.
56  *
57  * - If the arm64_ftr_bits[] for a register has a missing field, then this
58  *   field is treated as STRICT RES0, including for read_sanitised_ftr_reg().
59  *   This is stronger than FTR_HIDDEN and can be used to hide features from
60  *   KVM guests.
61  */
62 
63 #define pr_fmt(fmt) "CPU features: " fmt
64 
65 #include <linux/bsearch.h>
66 #include <linux/cpumask.h>
67 #include <linux/crash_dump.h>
68 #include <linux/kstrtox.h>
69 #include <linux/sort.h>
70 #include <linux/stop_machine.h>
71 #include <linux/sysfs.h>
72 #include <linux/types.h>
73 #include <linux/minmax.h>
74 #include <linux/mm.h>
75 #include <linux/cpu.h>
76 #include <linux/kasan.h>
77 #include <linux/percpu.h>
78 #include <linux/sched/isolation.h>
79 
80 #include <asm/cpu.h>
81 #include <asm/cpufeature.h>
82 #include <asm/cpu_ops.h>
83 #include <asm/fpsimd.h>
84 #include <asm/hwcap.h>
85 #include <asm/insn.h>
86 #include <asm/kvm_host.h>
87 #include <asm/mmu.h>
88 #include <asm/mmu_context.h>
89 #include <asm/mte.h>
90 #include <asm/hypervisor.h>
91 #include <asm/processor.h>
92 #include <asm/smp.h>
93 #include <asm/sysreg.h>
94 #include <asm/traps.h>
95 #include <asm/vectors.h>
96 #include <asm/virt.h>
97 
98 /* Kernel representation of AT_HWCAP and AT_HWCAP2 */
99 static DECLARE_BITMAP(elf_hwcap, MAX_CPU_FEATURES) __read_mostly;
100 
101 #ifdef CONFIG_COMPAT
102 #define COMPAT_ELF_HWCAP_DEFAULT	\
103 				(COMPAT_HWCAP_HALF|COMPAT_HWCAP_THUMB|\
104 				 COMPAT_HWCAP_FAST_MULT|COMPAT_HWCAP_EDSP|\
105 				 COMPAT_HWCAP_TLS|COMPAT_HWCAP_IDIV|\
106 				 COMPAT_HWCAP_LPAE)
107 unsigned int compat_elf_hwcap __read_mostly = COMPAT_ELF_HWCAP_DEFAULT;
108 unsigned int compat_elf_hwcap2 __read_mostly;
109 unsigned int compat_elf_hwcap3 __read_mostly;
110 #endif
111 
112 DECLARE_BITMAP(system_cpucaps, ARM64_NCAPS);
113 EXPORT_SYMBOL(system_cpucaps);
114 static struct arm64_cpu_capabilities const __ro_after_init *cpucap_ptrs[ARM64_NCAPS];
115 
116 DECLARE_BITMAP(boot_cpucaps, ARM64_NCAPS);
117 
118 /*
119  * arm64_use_ng_mappings must be placed in the .data section, otherwise it
120  * ends up in the .bss section where it is initialized in early_map_kernel()
121  * after the MMU (with the idmap) was enabled. create_init_idmap() - which
122  * runs before early_map_kernel() and reads the variable via PTE_MAYBE_NG -
123  * may end up generating an incorrect idmap page table attributes.
124  */
125 bool arm64_use_ng_mappings __read_mostly = false;
126 EXPORT_SYMBOL(arm64_use_ng_mappings);
127 
128 DEFINE_PER_CPU_READ_MOSTLY(const char *, this_cpu_vector) = vectors;
129 
130 /*
131  * Permit PER_LINUX32 and execve() of 32-bit binaries even if not all CPUs
132  * support it?
133  */
134 static bool __read_mostly allow_mismatched_32bit_el0;
135 
136 /*
137  * Static branch enabled only if allow_mismatched_32bit_el0 is set and we have
138  * seen at least one CPU capable of 32-bit EL0.
139  */
140 DEFINE_STATIC_KEY_FALSE(arm64_mismatched_32bit_el0);
141 
142 /*
143  * Mask of CPUs supporting 32-bit EL0.
144  * Only valid if arm64_mismatched_32bit_el0 is enabled.
145  */
146 static cpumask_var_t cpu_32bit_el0_mask __cpumask_var_read_mostly;
147 
dump_cpu_features(void)148 void dump_cpu_features(void)
149 {
150 	/* file-wide pr_fmt adds "CPU features: " prefix */
151 	pr_emerg("0x%*pb\n", ARM64_NCAPS, &system_cpucaps);
152 }
153 
154 #define __ARM64_MAX_POSITIVE(reg, field)				\
155 		((reg##_##field##_SIGNED ?				\
156 		  BIT(reg##_##field##_WIDTH - 1) :			\
157 		  BIT(reg##_##field##_WIDTH)) - 1)
158 
159 #define __ARM64_MIN_NEGATIVE(reg, field)  BIT(reg##_##field##_WIDTH - 1)
160 
161 #define __ARM64_CPUID_FIELDS(reg, field, min_value, max_value)		\
162 		.sys_reg = SYS_##reg,					\
163 		.field_pos = reg##_##field##_SHIFT,			\
164 		.field_width = reg##_##field##_WIDTH,			\
165 		.sign = reg##_##field##_SIGNED,				\
166 		.min_field_value = min_value,				\
167 		.max_field_value = max_value,
168 
169 /*
170  * ARM64_CPUID_FIELDS() encodes a field with a range from min_value to
171  * an implicit maximum that depends on the sign-ess of the field.
172  *
173  * An unsigned field will be capped at all ones, while a signed field
174  * will be limited to the positive half only.
175  */
176 #define ARM64_CPUID_FIELDS(reg, field, min_value)			\
177 	__ARM64_CPUID_FIELDS(reg, field,				\
178 			     SYS_FIELD_VALUE(reg, field, min_value),	\
179 			     __ARM64_MAX_POSITIVE(reg, field))
180 
181 /*
182  * ARM64_CPUID_FIELDS_NEG() encodes a field with a range from an
183  * implicit minimal value to max_value. This should be used when
184  * matching a non-implemented property.
185  */
186 #define ARM64_CPUID_FIELDS_NEG(reg, field, max_value)			\
187 	__ARM64_CPUID_FIELDS(reg, field,				\
188 			     __ARM64_MIN_NEGATIVE(reg, field),		\
189 			     SYS_FIELD_VALUE(reg, field, max_value))
190 
191 #define __ARM64_FTR_BITS(SIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
192 	{						\
193 		.sign = SIGNED,				\
194 		.visible = VISIBLE,			\
195 		.strict = STRICT,			\
196 		.type = TYPE,				\
197 		.shift = SHIFT,				\
198 		.width = WIDTH,				\
199 		.safe_val = SAFE_VAL,			\
200 	}
201 
202 /* Define a feature with unsigned values */
203 #define ARM64_FTR_BITS(VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
204 	__ARM64_FTR_BITS(FTR_UNSIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL)
205 
206 /* Define a feature with a signed value */
207 #define S_ARM64_FTR_BITS(VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL) \
208 	__ARM64_FTR_BITS(FTR_SIGNED, VISIBLE, STRICT, TYPE, SHIFT, WIDTH, SAFE_VAL)
209 
210 #define ARM64_FTR_END					\
211 	{						\
212 		.width = 0,				\
213 	}
214 
215 static void cpu_enable_cnp(struct arm64_cpu_capabilities const *cap);
216 
217 static bool __system_matches_cap(unsigned int n);
218 
219 /*
220  * NOTE: Any changes to the visibility of features should be kept in
221  * sync with the documentation of the CPU feature register ABI.
222  */
223 static const struct arm64_ftr_bits ftr_id_aa64isar0[] = {
224 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_RNDR_SHIFT, 4, 0),
225 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_TLB_SHIFT, 4, 0),
226 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_TS_SHIFT, 4, 0),
227 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_FHM_SHIFT, 4, 0),
228 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_DP_SHIFT, 4, 0),
229 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_SM4_SHIFT, 4, 0),
230 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_SM3_SHIFT, 4, 0),
231 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_SHA3_SHIFT, 4, 0),
232 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_RDM_SHIFT, 4, 0),
233 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_ATOMIC_SHIFT, 4, 0),
234 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_CRC32_SHIFT, 4, 0),
235 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_SHA2_SHIFT, 4, 0),
236 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_SHA1_SHIFT, 4, 0),
237 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR0_EL1_AES_SHIFT, 4, 0),
238 	ARM64_FTR_END,
239 };
240 
241 static const struct arm64_ftr_bits ftr_id_aa64isar1[] = {
242 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_XS_SHIFT, 4, 0),
243 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_I8MM_SHIFT, 4, 0),
244 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_DGH_SHIFT, 4, 0),
245 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_BF16_SHIFT, 4, 0),
246 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_SPECRES_SHIFT, 4, 0),
247 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_SB_SHIFT, 4, 0),
248 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_FRINTTS_SHIFT, 4, 0),
249 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH),
250 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_GPI_SHIFT, 4, 0),
251 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH),
252 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_GPA_SHIFT, 4, 0),
253 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_LRCPC_SHIFT, 4, 0),
254 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_FCMA_SHIFT, 4, 0),
255 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_JSCVT_SHIFT, 4, 0),
256 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH),
257 		       FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_EL1_API_SHIFT, 4, 0),
258 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH),
259 		       FTR_STRICT, FTR_EXACT, ID_AA64ISAR1_EL1_APA_SHIFT, 4, 0),
260 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR1_EL1_DPB_SHIFT, 4, 0),
261 	ARM64_FTR_END,
262 };
263 
264 static const struct arm64_ftr_bits ftr_id_aa64isar2[] = {
265 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_LUT_SHIFT, 4, 0),
266 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_CSSC_SHIFT, 4, 0),
267 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_RPRFM_SHIFT, 4, 0),
268 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_CLRBHB_SHIFT, 4, 0),
269 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_BC_SHIFT, 4, 0),
270 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_MOPS_SHIFT, 4, 0),
271 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH),
272 		       FTR_STRICT, FTR_EXACT, ID_AA64ISAR2_EL1_APA3_SHIFT, 4, 0),
273 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_PTR_AUTH),
274 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_GPA3_SHIFT, 4, 0),
275 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_RPRES_SHIFT, 4, 0),
276 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR2_EL1_WFxT_SHIFT, 4, 0),
277 	ARM64_FTR_END,
278 };
279 
280 static const struct arm64_ftr_bits ftr_id_aa64isar3[] = {
281 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR3_EL1_FPRCVT_SHIFT, 4, 0),
282 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64ISAR3_EL1_FAMINMAX_SHIFT, 4, 0),
283 	ARM64_FTR_END,
284 };
285 
286 static const struct arm64_ftr_bits ftr_id_aa64pfr0[] = {
287 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_CSV3_SHIFT, 4, 0),
288 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_CSV2_SHIFT, 4, 0),
289 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_DIT_SHIFT, 4, 0),
290 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_AMU_SHIFT, 4, 0),
291 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_MPAM_SHIFT, 4, 0),
292 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_SEL2_SHIFT, 4, 0),
293 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
294 				   FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_SVE_SHIFT, 4, 0),
295 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_RAS_SHIFT, 4, 0),
296 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_GIC_SHIFT, 4, 0),
297 	S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_AdvSIMD_SHIFT, 4, ID_AA64PFR0_EL1_AdvSIMD_NI),
298 	S_ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_FP_SHIFT, 4, ID_AA64PFR0_EL1_FP_NI),
299 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_EL3_SHIFT, 4, 0),
300 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_EL2_SHIFT, 4, 0),
301 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_EL1_SHIFT, 4, ID_AA64PFR0_EL1_EL1_IMP),
302 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR0_EL1_EL0_SHIFT, 4, ID_AA64PFR0_EL1_EL0_IMP),
303 	ARM64_FTR_END,
304 };
305 
306 static const struct arm64_ftr_bits ftr_id_aa64pfr1[] = {
307 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_DF2_SHIFT, 4, 0),
308 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_GCS),
309 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_GCS_SHIFT, 4, 0),
310 	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_MTE_frac_SHIFT, 4, 0),
311 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
312 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_SME_SHIFT, 4, 0),
313 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_MPAM_frac_SHIFT, 4, 0),
314 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_RAS_frac_SHIFT, 4, 0),
315 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_MTE),
316 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_MTE_SHIFT, 4, ID_AA64PFR1_EL1_MTE_NI),
317 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_SSBS_SHIFT, 4, ID_AA64PFR1_EL1_SSBS_NI),
318 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_BTI),
319 				    FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR1_EL1_BT_SHIFT, 4, 0),
320 	ARM64_FTR_END,
321 };
322 
323 static const struct arm64_ftr_bits ftr_id_aa64pfr2[] = {
324 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64PFR2_EL1_FPMR_SHIFT, 4, 0),
325 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR2_EL1_MTEFAR_SHIFT, 4, ID_AA64PFR2_EL1_MTEFAR_NI),
326 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64PFR2_EL1_MTESTOREONLY_SHIFT, 4, ID_AA64PFR2_EL1_MTESTOREONLY_NI),
327 	ARM64_FTR_END,
328 };
329 
330 static const struct arm64_ftr_bits ftr_id_aa64zfr0[] = {
331 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
332 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_F64MM_SHIFT, 4, 0),
333 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
334 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_F32MM_SHIFT, 4, 0),
335 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
336 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_F16MM_SHIFT, 4, 0),
337 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
338 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_I8MM_SHIFT, 4, 0),
339 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
340 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_SM4_SHIFT, 4, 0),
341 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
342 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_SHA3_SHIFT, 4, 0),
343 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
344 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_B16B16_SHIFT, 4, 0),
345 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
346 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_BF16_SHIFT, 4, 0),
347 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
348 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_BitPerm_SHIFT, 4, 0),
349 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
350 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_EltPerm_SHIFT, 4, 0),
351 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
352 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_AES_SHIFT, 4, 0),
353 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SVE),
354 		       FTR_STRICT, FTR_LOWER_SAFE, ID_AA64ZFR0_EL1_SVEver_SHIFT, 4, 0),
355 	ARM64_FTR_END,
356 };
357 
358 static const struct arm64_ftr_bits ftr_id_aa64smfr0[] = {
359 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
360 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_FA64_SHIFT, 1, 0),
361 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
362 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_LUTv2_SHIFT, 1, 0),
363 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
364 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_SMEver_SHIFT, 4, 0),
365 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
366 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_I16I64_SHIFT, 4, 0),
367 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
368 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_F64F64_SHIFT, 1, 0),
369 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
370 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_I16I32_SHIFT, 4, 0),
371 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
372 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_B16B16_SHIFT, 1, 0),
373 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
374 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_F16F16_SHIFT, 1, 0),
375 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
376 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_F8F16_SHIFT, 1, 0),
377 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
378 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_F8F32_SHIFT, 1, 0),
379 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
380 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_I8I32_SHIFT, 4, 0),
381 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
382 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_F16F32_SHIFT, 1, 0),
383 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
384 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_B16F32_SHIFT, 1, 0),
385 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
386 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_BI32I32_SHIFT, 1, 0),
387 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
388 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_F32F32_SHIFT, 1, 0),
389 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
390 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_SF8FMA_SHIFT, 1, 0),
391 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
392 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_SF8DP4_SHIFT, 1, 0),
393 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
394 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_SF8DP2_SHIFT, 1, 0),
395 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
396 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_SBitPerm_SHIFT, 1, 0),
397 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
398 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_AES_SHIFT, 1, 0),
399 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
400 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_SFEXPA_SHIFT, 1, 0),
401 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
402 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_STMOP_SHIFT, 1, 0),
403 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_SME),
404 		       FTR_STRICT, FTR_EXACT, ID_AA64SMFR0_EL1_SMOP4_SHIFT, 1, 0),
405 	ARM64_FTR_END,
406 };
407 
408 static const struct arm64_ftr_bits ftr_id_aa64fpfr0[] = {
409 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8CVT_SHIFT, 1, 0),
410 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8FMA_SHIFT, 1, 0),
411 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8DP4_SHIFT, 1, 0),
412 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8DP2_SHIFT, 1, 0),
413 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8MM8_SHIFT, 1, 0),
414 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8MM4_SHIFT, 1, 0),
415 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8E4M3_SHIFT, 1, 0),
416 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, ID_AA64FPFR0_EL1_F8E5M2_SHIFT, 1, 0),
417 	ARM64_FTR_END,
418 };
419 
420 static const struct arm64_ftr_bits ftr_id_aa64mmfr0[] = {
421 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_ECV_SHIFT, 4, 0),
422 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_FGT_SHIFT, 4, 0),
423 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_EXS_SHIFT, 4, 0),
424 	/*
425 	 * Page size not being supported at Stage-2 is not fatal. You
426 	 * just give up KVM if PAGE_SIZE isn't supported there. Go fix
427 	 * your favourite nesting hypervisor.
428 	 *
429 	 * There is a small corner case where the hypervisor explicitly
430 	 * advertises a given granule size at Stage-2 (value 2) on some
431 	 * vCPUs, and uses the fallback to Stage-1 (value 0) for other
432 	 * vCPUs. Although this is not forbidden by the architecture, it
433 	 * indicates that the hypervisor is being silly (or buggy).
434 	 *
435 	 * We make no effort to cope with this and pretend that if these
436 	 * fields are inconsistent across vCPUs, then it isn't worth
437 	 * trying to bring KVM up.
438 	 */
439 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64MMFR0_EL1_TGRAN4_2_SHIFT, 4, 1),
440 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64MMFR0_EL1_TGRAN64_2_SHIFT, 4, 1),
441 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64MMFR0_EL1_TGRAN16_2_SHIFT, 4, 1),
442 	/*
443 	 * We already refuse to boot CPUs that don't support our configured
444 	 * page size, so we can only detect mismatches for a page size other
445 	 * than the one we're currently using. Unfortunately, SoCs like this
446 	 * exist in the wild so, even though we don't like it, we'll have to go
447 	 * along with it and treat them as non-strict.
448 	 */
449 	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_TGRAN4_SHIFT, 4, ID_AA64MMFR0_EL1_TGRAN4_NI),
450 	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_TGRAN64_SHIFT, 4, ID_AA64MMFR0_EL1_TGRAN64_NI),
451 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_TGRAN16_SHIFT, 4, ID_AA64MMFR0_EL1_TGRAN16_NI),
452 
453 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_BIGENDEL0_SHIFT, 4, 0),
454 	/* Linux shouldn't care about secure memory */
455 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_SNSMEM_SHIFT, 4, 0),
456 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_BIGEND_SHIFT, 4, 0),
457 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_ASIDBITS_SHIFT, 4, 0),
458 	/*
459 	 * Differing PARange is fine as long as all peripherals and memory are mapped
460 	 * within the minimum PARange of all CPUs
461 	 */
462 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR0_EL1_PARANGE_SHIFT, 4, 0),
463 	ARM64_FTR_END,
464 };
465 
466 static const struct arm64_ftr_bits ftr_id_aa64mmfr1[] = {
467 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_ECBHB_SHIFT, 4, 0),
468 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_TIDCP1_SHIFT, 4, 0),
469 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_AFP_SHIFT, 4, 0),
470 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_HCX_SHIFT, 4, 0),
471 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_ETS_SHIFT, 4, 0),
472 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_TWED_SHIFT, 4, 0),
473 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_XNX_SHIFT, 4, 0),
474 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_AA64MMFR1_EL1_SpecSEI_SHIFT, 4, 0),
475 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_PAN_SHIFT, 4, 0),
476 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_LO_SHIFT, 4, 0),
477 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_HPDS_SHIFT, 4, 0),
478 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_VH_SHIFT, 4, 0),
479 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_VMIDBits_SHIFT, 4, 0),
480 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR1_EL1_HAFDBS_SHIFT, 4, 0),
481 	ARM64_FTR_END,
482 };
483 
484 static const struct arm64_ftr_bits ftr_id_aa64mmfr2[] = {
485 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_E0PD_SHIFT, 4, 0),
486 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_EVT_SHIFT, 4, 0),
487 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_BBM_SHIFT, 4, 0),
488 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_TTL_SHIFT, 4, 0),
489 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_FWB_SHIFT, 4, 0),
490 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_IDS_SHIFT, 4, 0),
491 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_AT_SHIFT, 4, 0),
492 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_ST_SHIFT, 4, 0),
493 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_NV_SHIFT, 4, 0),
494 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_CCIDX_SHIFT, 4, 0),
495 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_VARange_SHIFT, 4, 0),
496 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_IESB_SHIFT, 4, 0),
497 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_LSM_SHIFT, 4, 0),
498 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_UAO_SHIFT, 4, 0),
499 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR2_EL1_CnP_SHIFT, 4, 0),
500 	ARM64_FTR_END,
501 };
502 
503 static const struct arm64_ftr_bits ftr_id_aa64mmfr3[] = {
504 	ARM64_FTR_BITS(FTR_VISIBLE_IF_IS_ENABLED(CONFIG_ARM64_POE),
505 		       FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR3_EL1_S1POE_SHIFT, 4, 0),
506 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR3_EL1_S1PIE_SHIFT, 4, 0),
507 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR3_EL1_SCTLRX_SHIFT, 4, 0),
508 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64MMFR3_EL1_TCRX_SHIFT, 4, 0),
509 	ARM64_FTR_END,
510 };
511 
512 static const struct arm64_ftr_bits ftr_id_aa64mmfr4[] = {
513 	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR4_EL1_E2H0_SHIFT, 4, 0),
514 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64MMFR4_EL1_NV_frac_SHIFT, 4, 0),
515 	ARM64_FTR_END,
516 };
517 
518 static const struct arm64_ftr_bits ftr_ctr[] = {
519 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, 31, 1, 1), /* RES1 */
520 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_EL0_DIC_SHIFT, 1, 1),
521 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_EL0_IDC_SHIFT, 1, 1),
522 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_HIGHER_OR_ZERO_SAFE, CTR_EL0_CWG_SHIFT, 4, 0),
523 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_HIGHER_OR_ZERO_SAFE, CTR_EL0_ERG_SHIFT, 4, 0),
524 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_EL0_DminLine_SHIFT, 4, 1),
525 	/*
526 	 * Linux can handle differing I-cache policies. Userspace JITs will
527 	 * make use of *minLine.
528 	 * If we have differing I-cache policies, report it as the weakest - VIPT.
529 	 */
530 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_EXACT, CTR_EL0_L1Ip_SHIFT, 2, CTR_EL0_L1Ip_VIPT),	/* L1Ip */
531 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, CTR_EL0_IminLine_SHIFT, 4, 0),
532 	ARM64_FTR_END,
533 };
534 
535 static struct arm64_ftr_override __ro_after_init no_override = { };
536 
537 struct arm64_ftr_reg arm64_ftr_reg_ctrel0 = {
538 	.name		= "SYS_CTR_EL0",
539 	.ftr_bits	= ftr_ctr,
540 	.override	= &no_override,
541 };
542 
543 static const struct arm64_ftr_bits ftr_id_mmfr0[] = {
544 	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_InnerShr_SHIFT, 4, 0xf),
545 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_FCSE_SHIFT, 4, 0),
546 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_AuxReg_SHIFT, 4, 0),
547 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_TCM_SHIFT, 4, 0),
548 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_ShareLvl_SHIFT, 4, 0),
549 	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_OuterShr_SHIFT, 4, 0xf),
550 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_PMSA_SHIFT, 4, 0),
551 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR0_EL1_VMSA_SHIFT, 4, 0),
552 	ARM64_FTR_END,
553 };
554 
555 static const struct arm64_ftr_bits ftr_id_aa64dfr0[] = {
556 	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_EL1_DoubleLock_SHIFT, 4, 0),
557 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_AA64DFR0_EL1_PMSVer_SHIFT, 4, 0),
558 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_EL1_CTX_CMPs_SHIFT, 4, 0),
559 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_EL1_WRPs_SHIFT, 4, 0),
560 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_AA64DFR0_EL1_BRPs_SHIFT, 4, 0),
561 	/*
562 	 * We can instantiate multiple PMU instances with different levels
563 	 * of support.
564 	 */
565 	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_AA64DFR0_EL1_PMUVer_SHIFT, 4, 0),
566 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, ID_AA64DFR0_EL1_DebugVer_SHIFT, 4, 0x6),
567 	ARM64_FTR_END,
568 };
569 
570 static const struct arm64_ftr_bits ftr_mvfr0[] = {
571 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPRound_SHIFT, 4, 0),
572 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPShVec_SHIFT, 4, 0),
573 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPSqrt_SHIFT, 4, 0),
574 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPDivide_SHIFT, 4, 0),
575 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPTrap_SHIFT, 4, 0),
576 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPDP_SHIFT, 4, 0),
577 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_FPSP_SHIFT, 4, 0),
578 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR0_EL1_SIMDReg_SHIFT, 4, 0),
579 	ARM64_FTR_END,
580 };
581 
582 static const struct arm64_ftr_bits ftr_mvfr1[] = {
583 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDFMAC_SHIFT, 4, 0),
584 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_FPHP_SHIFT, 4, 0),
585 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDHP_SHIFT, 4, 0),
586 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDSP_SHIFT, 4, 0),
587 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDInt_SHIFT, 4, 0),
588 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_SIMDLS_SHIFT, 4, 0),
589 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_FPDNaN_SHIFT, 4, 0),
590 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR1_EL1_FPFtZ_SHIFT, 4, 0),
591 	ARM64_FTR_END,
592 };
593 
594 static const struct arm64_ftr_bits ftr_mvfr2[] = {
595 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR2_EL1_FPMisc_SHIFT, 4, 0),
596 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MVFR2_EL1_SIMDMisc_SHIFT, 4, 0),
597 	ARM64_FTR_END,
598 };
599 
600 static const struct arm64_ftr_bits ftr_dczid[] = {
601 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_EXACT, DCZID_EL0_DZP_SHIFT, 1, 1),
602 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, DCZID_EL0_BS_SHIFT, 4, 0),
603 	ARM64_FTR_END,
604 };
605 
606 static const struct arm64_ftr_bits ftr_gmid[] = {
607 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, GMID_EL1_BS_SHIFT, 4, 0),
608 	ARM64_FTR_END,
609 };
610 
611 static const struct arm64_ftr_bits ftr_id_isar0[] = {
612 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_Divide_SHIFT, 4, 0),
613 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_Debug_SHIFT, 4, 0),
614 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_Coproc_SHIFT, 4, 0),
615 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_CmpBranch_SHIFT, 4, 0),
616 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_BitField_SHIFT, 4, 0),
617 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_BitCount_SHIFT, 4, 0),
618 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR0_EL1_Swap_SHIFT, 4, 0),
619 	ARM64_FTR_END,
620 };
621 
622 static const struct arm64_ftr_bits ftr_id_isar5[] = {
623 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_RDM_SHIFT, 4, 0),
624 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_CRC32_SHIFT, 4, 0),
625 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_SHA2_SHIFT, 4, 0),
626 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_SHA1_SHIFT, 4, 0),
627 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_AES_SHIFT, 4, 0),
628 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR5_EL1_SEVL_SHIFT, 4, 0),
629 	ARM64_FTR_END,
630 };
631 
632 static const struct arm64_ftr_bits ftr_id_mmfr4[] = {
633 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_EVT_SHIFT, 4, 0),
634 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_CCIDX_SHIFT, 4, 0),
635 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_LSM_SHIFT, 4, 0),
636 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_HPDS_SHIFT, 4, 0),
637 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_CnP_SHIFT, 4, 0),
638 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_XNX_SHIFT, 4, 0),
639 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR4_EL1_AC2_SHIFT, 4, 0),
640 
641 	/*
642 	 * SpecSEI = 1 indicates that the PE might generate an SError on an
643 	 * external abort on speculative read. It is safe to assume that an
644 	 * SError might be generated than it will not be. Hence it has been
645 	 * classified as FTR_HIGHER_SAFE.
646 	 */
647 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_HIGHER_SAFE, ID_MMFR4_EL1_SpecSEI_SHIFT, 4, 0),
648 	ARM64_FTR_END,
649 };
650 
651 static const struct arm64_ftr_bits ftr_id_isar4[] = {
652 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_SWP_frac_SHIFT, 4, 0),
653 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_PSR_M_SHIFT, 4, 0),
654 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_SynchPrim_frac_SHIFT, 4, 0),
655 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_Barrier_SHIFT, 4, 0),
656 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_SMC_SHIFT, 4, 0),
657 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_Writeback_SHIFT, 4, 0),
658 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_WithShifts_SHIFT, 4, 0),
659 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR4_EL1_Unpriv_SHIFT, 4, 0),
660 	ARM64_FTR_END,
661 };
662 
663 static const struct arm64_ftr_bits ftr_id_mmfr5[] = {
664 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_MMFR5_EL1_ETS_SHIFT, 4, 0),
665 	ARM64_FTR_END,
666 };
667 
668 static const struct arm64_ftr_bits ftr_id_isar6[] = {
669 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_I8MM_SHIFT, 4, 0),
670 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_BF16_SHIFT, 4, 0),
671 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_SPECRES_SHIFT, 4, 0),
672 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_SB_SHIFT, 4, 0),
673 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_FHM_SHIFT, 4, 0),
674 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_DP_SHIFT, 4, 0),
675 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_ISAR6_EL1_JSCVT_SHIFT, 4, 0),
676 	ARM64_FTR_END,
677 };
678 
679 static const struct arm64_ftr_bits ftr_id_pfr0[] = {
680 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_DIT_SHIFT, 4, 0),
681 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_CSV2_SHIFT, 4, 0),
682 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_State3_SHIFT, 4, 0),
683 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_State2_SHIFT, 4, 0),
684 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_State1_SHIFT, 4, 0),
685 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR0_EL1_State0_SHIFT, 4, 0),
686 	ARM64_FTR_END,
687 };
688 
689 static const struct arm64_ftr_bits ftr_id_pfr1[] = {
690 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_GIC_SHIFT, 4, 0),
691 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_Virt_frac_SHIFT, 4, 0),
692 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_Sec_frac_SHIFT, 4, 0),
693 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_GenTimer_SHIFT, 4, 0),
694 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_Virtualization_SHIFT, 4, 0),
695 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_MProgMod_SHIFT, 4, 0),
696 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_Security_SHIFT, 4, 0),
697 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_PFR1_EL1_ProgMod_SHIFT, 4, 0),
698 	ARM64_FTR_END,
699 };
700 
701 static const struct arm64_ftr_bits ftr_id_pfr2[] = {
702 	ARM64_FTR_BITS(FTR_VISIBLE, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR2_EL1_SSBS_SHIFT, 4, 0),
703 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, ID_PFR2_EL1_CSV3_SHIFT, 4, 0),
704 	ARM64_FTR_END,
705 };
706 
707 static const struct arm64_ftr_bits ftr_id_dfr0[] = {
708 	/* [31:28] TraceFilt */
709 	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_EXACT, ID_DFR0_EL1_PerfMon_SHIFT, 4, 0),
710 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_MProfDbg_SHIFT, 4, 0),
711 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_MMapTrc_SHIFT, 4, 0),
712 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_CopTrc_SHIFT, 4, 0),
713 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_MMapDbg_SHIFT, 4, 0),
714 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_CopSDbg_SHIFT, 4, 0),
715 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR0_EL1_CopDbg_SHIFT, 4, 0),
716 	ARM64_FTR_END,
717 };
718 
719 static const struct arm64_ftr_bits ftr_id_dfr1[] = {
720 	S_ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, ID_DFR1_EL1_MTPMU_SHIFT, 4, 0),
721 	ARM64_FTR_END,
722 };
723 
724 static const struct arm64_ftr_bits ftr_mpamidr[] = {
725 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, MPAMIDR_EL1_PMG_MAX_SHIFT, MPAMIDR_EL1_PMG_MAX_WIDTH, 0),
726 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, MPAMIDR_EL1_VPMR_MAX_SHIFT, MPAMIDR_EL1_VPMR_MAX_WIDTH, 0),
727 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, MPAMIDR_EL1_HAS_HCR_SHIFT, 1, 0),
728 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_NONSTRICT, FTR_LOWER_SAFE, MPAMIDR_EL1_PARTID_MAX_SHIFT, MPAMIDR_EL1_PARTID_MAX_WIDTH, 0),
729 	ARM64_FTR_END,
730 };
731 
732 /*
733  * Common ftr bits for a 32bit register with all hidden, strict
734  * attributes, with 4bit feature fields and a default safe value of
735  * 0. Covers the following 32bit registers:
736  * id_isar[1-3], id_mmfr[1-3]
737  */
738 static const struct arm64_ftr_bits ftr_generic_32bits[] = {
739 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 28, 4, 0),
740 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 24, 4, 0),
741 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 20, 4, 0),
742 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 16, 4, 0),
743 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 12, 4, 0),
744 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 8, 4, 0),
745 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 4, 4, 0),
746 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_LOWER_SAFE, 0, 4, 0),
747 	ARM64_FTR_END,
748 };
749 
750 /* Table for a single 32bit feature value */
751 static const struct arm64_ftr_bits ftr_single32[] = {
752 	ARM64_FTR_BITS(FTR_HIDDEN, FTR_STRICT, FTR_EXACT, 0, 32, 0),
753 	ARM64_FTR_END,
754 };
755 
756 static const struct arm64_ftr_bits ftr_raz[] = {
757 	ARM64_FTR_END,
758 };
759 
760 #define __ARM64_FTR_REG_OVERRIDE(id_str, id, table, ovr) {	\
761 		.sys_id = id,					\
762 		.reg = 	&(struct arm64_ftr_reg){		\
763 			.name = id_str,				\
764 			.override = (ovr),			\
765 			.ftr_bits = &((table)[0]),		\
766 	}}
767 
768 #define ARM64_FTR_REG_OVERRIDE(id, table, ovr)	\
769 	__ARM64_FTR_REG_OVERRIDE(#id, id, table, ovr)
770 
771 #define ARM64_FTR_REG(id, table)		\
772 	__ARM64_FTR_REG_OVERRIDE(#id, id, table, &no_override)
773 
774 struct arm64_ftr_override __read_mostly id_aa64mmfr0_override;
775 struct arm64_ftr_override __read_mostly id_aa64mmfr1_override;
776 struct arm64_ftr_override __read_mostly id_aa64mmfr2_override;
777 struct arm64_ftr_override __read_mostly id_aa64pfr0_override;
778 struct arm64_ftr_override __read_mostly id_aa64pfr1_override;
779 struct arm64_ftr_override __read_mostly id_aa64zfr0_override;
780 struct arm64_ftr_override __read_mostly id_aa64smfr0_override;
781 struct arm64_ftr_override __read_mostly id_aa64isar1_override;
782 struct arm64_ftr_override __read_mostly id_aa64isar2_override;
783 
784 struct arm64_ftr_override __read_mostly arm64_sw_feature_override;
785 
786 static const struct __ftr_reg_entry {
787 	u32			sys_id;
788 	struct arm64_ftr_reg 	*reg;
789 } arm64_ftr_regs[] = {
790 
791 	/* Op1 = 0, CRn = 0, CRm = 1 */
792 	ARM64_FTR_REG(SYS_ID_PFR0_EL1, ftr_id_pfr0),
793 	ARM64_FTR_REG(SYS_ID_PFR1_EL1, ftr_id_pfr1),
794 	ARM64_FTR_REG(SYS_ID_DFR0_EL1, ftr_id_dfr0),
795 	ARM64_FTR_REG(SYS_ID_MMFR0_EL1, ftr_id_mmfr0),
796 	ARM64_FTR_REG(SYS_ID_MMFR1_EL1, ftr_generic_32bits),
797 	ARM64_FTR_REG(SYS_ID_MMFR2_EL1, ftr_generic_32bits),
798 	ARM64_FTR_REG(SYS_ID_MMFR3_EL1, ftr_generic_32bits),
799 
800 	/* Op1 = 0, CRn = 0, CRm = 2 */
801 	ARM64_FTR_REG(SYS_ID_ISAR0_EL1, ftr_id_isar0),
802 	ARM64_FTR_REG(SYS_ID_ISAR1_EL1, ftr_generic_32bits),
803 	ARM64_FTR_REG(SYS_ID_ISAR2_EL1, ftr_generic_32bits),
804 	ARM64_FTR_REG(SYS_ID_ISAR3_EL1, ftr_generic_32bits),
805 	ARM64_FTR_REG(SYS_ID_ISAR4_EL1, ftr_id_isar4),
806 	ARM64_FTR_REG(SYS_ID_ISAR5_EL1, ftr_id_isar5),
807 	ARM64_FTR_REG(SYS_ID_MMFR4_EL1, ftr_id_mmfr4),
808 	ARM64_FTR_REG(SYS_ID_ISAR6_EL1, ftr_id_isar6),
809 
810 	/* Op1 = 0, CRn = 0, CRm = 3 */
811 	ARM64_FTR_REG(SYS_MVFR0_EL1, ftr_mvfr0),
812 	ARM64_FTR_REG(SYS_MVFR1_EL1, ftr_mvfr1),
813 	ARM64_FTR_REG(SYS_MVFR2_EL1, ftr_mvfr2),
814 	ARM64_FTR_REG(SYS_ID_PFR2_EL1, ftr_id_pfr2),
815 	ARM64_FTR_REG(SYS_ID_DFR1_EL1, ftr_id_dfr1),
816 	ARM64_FTR_REG(SYS_ID_MMFR5_EL1, ftr_id_mmfr5),
817 
818 	/* Op1 = 0, CRn = 0, CRm = 4 */
819 	ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64PFR0_EL1, ftr_id_aa64pfr0,
820 			       &id_aa64pfr0_override),
821 	ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64PFR1_EL1, ftr_id_aa64pfr1,
822 			       &id_aa64pfr1_override),
823 	ARM64_FTR_REG(SYS_ID_AA64PFR2_EL1, ftr_id_aa64pfr2),
824 	ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64ZFR0_EL1, ftr_id_aa64zfr0,
825 			       &id_aa64zfr0_override),
826 	ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64SMFR0_EL1, ftr_id_aa64smfr0,
827 			       &id_aa64smfr0_override),
828 	ARM64_FTR_REG(SYS_ID_AA64FPFR0_EL1, ftr_id_aa64fpfr0),
829 
830 	/* Op1 = 0, CRn = 0, CRm = 5 */
831 	ARM64_FTR_REG(SYS_ID_AA64DFR0_EL1, ftr_id_aa64dfr0),
832 	ARM64_FTR_REG(SYS_ID_AA64DFR1_EL1, ftr_raz),
833 
834 	/* Op1 = 0, CRn = 0, CRm = 6 */
835 	ARM64_FTR_REG(SYS_ID_AA64ISAR0_EL1, ftr_id_aa64isar0),
836 	ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64ISAR1_EL1, ftr_id_aa64isar1,
837 			       &id_aa64isar1_override),
838 	ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64ISAR2_EL1, ftr_id_aa64isar2,
839 			       &id_aa64isar2_override),
840 	ARM64_FTR_REG(SYS_ID_AA64ISAR3_EL1, ftr_id_aa64isar3),
841 
842 	/* Op1 = 0, CRn = 0, CRm = 7 */
843 	ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64MMFR0_EL1, ftr_id_aa64mmfr0,
844 			       &id_aa64mmfr0_override),
845 	ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64MMFR1_EL1, ftr_id_aa64mmfr1,
846 			       &id_aa64mmfr1_override),
847 	ARM64_FTR_REG_OVERRIDE(SYS_ID_AA64MMFR2_EL1, ftr_id_aa64mmfr2,
848 			       &id_aa64mmfr2_override),
849 	ARM64_FTR_REG(SYS_ID_AA64MMFR3_EL1, ftr_id_aa64mmfr3),
850 	ARM64_FTR_REG(SYS_ID_AA64MMFR4_EL1, ftr_id_aa64mmfr4),
851 
852 	/* Op1 = 0, CRn = 10, CRm = 4 */
853 	ARM64_FTR_REG(SYS_MPAMIDR_EL1, ftr_mpamidr),
854 
855 	/* Op1 = 1, CRn = 0, CRm = 0 */
856 	ARM64_FTR_REG(SYS_GMID_EL1, ftr_gmid),
857 
858 	/* Op1 = 3, CRn = 0, CRm = 0 */
859 	{ SYS_CTR_EL0, &arm64_ftr_reg_ctrel0 },
860 	ARM64_FTR_REG(SYS_DCZID_EL0, ftr_dczid),
861 
862 	/* Op1 = 3, CRn = 14, CRm = 0 */
863 	ARM64_FTR_REG(SYS_CNTFRQ_EL0, ftr_single32),
864 };
865 
search_cmp_ftr_reg(const void * id,const void * regp)866 static int search_cmp_ftr_reg(const void *id, const void *regp)
867 {
868 	return (int)(unsigned long)id - (int)((const struct __ftr_reg_entry *)regp)->sys_id;
869 }
870 
871 /*
872  * get_arm64_ftr_reg_nowarn - Looks up a feature register entry using
873  * its sys_reg() encoding. With the array arm64_ftr_regs sorted in the
874  * ascending order of sys_id, we use binary search to find a matching
875  * entry.
876  *
877  * returns - Upon success,  matching ftr_reg entry for id.
878  *         - NULL on failure. It is upto the caller to decide
879  *	     the impact of a failure.
880  */
get_arm64_ftr_reg_nowarn(u32 sys_id)881 static struct arm64_ftr_reg *get_arm64_ftr_reg_nowarn(u32 sys_id)
882 {
883 	const struct __ftr_reg_entry *ret;
884 
885 	ret = bsearch((const void *)(unsigned long)sys_id,
886 			arm64_ftr_regs,
887 			ARRAY_SIZE(arm64_ftr_regs),
888 			sizeof(arm64_ftr_regs[0]),
889 			search_cmp_ftr_reg);
890 	if (ret)
891 		return ret->reg;
892 	return NULL;
893 }
894 
895 /*
896  * get_arm64_ftr_reg - Looks up a feature register entry using
897  * its sys_reg() encoding. This calls get_arm64_ftr_reg_nowarn().
898  *
899  * returns - Upon success,  matching ftr_reg entry for id.
900  *         - NULL on failure but with an WARN_ON().
901  */
get_arm64_ftr_reg(u32 sys_id)902 struct arm64_ftr_reg *get_arm64_ftr_reg(u32 sys_id)
903 {
904 	struct arm64_ftr_reg *reg;
905 
906 	reg = get_arm64_ftr_reg_nowarn(sys_id);
907 
908 	/*
909 	 * Requesting a non-existent register search is an error. Warn
910 	 * and let the caller handle it.
911 	 */
912 	WARN_ON(!reg);
913 	return reg;
914 }
915 
arm64_ftr_set_value(const struct arm64_ftr_bits * ftrp,s64 reg,s64 ftr_val)916 static u64 arm64_ftr_set_value(const struct arm64_ftr_bits *ftrp, s64 reg,
917 			       s64 ftr_val)
918 {
919 	u64 mask = arm64_ftr_mask(ftrp);
920 
921 	reg &= ~mask;
922 	reg |= (ftr_val << ftrp->shift) & mask;
923 	return reg;
924 }
925 
arm64_ftr_safe_value(const struct arm64_ftr_bits * ftrp,s64 new,s64 cur)926 s64 arm64_ftr_safe_value(const struct arm64_ftr_bits *ftrp, s64 new,
927 				s64 cur)
928 {
929 	s64 ret = 0;
930 
931 	switch (ftrp->type) {
932 	case FTR_EXACT:
933 		ret = ftrp->safe_val;
934 		break;
935 	case FTR_LOWER_SAFE:
936 		ret = min(new, cur);
937 		break;
938 	case FTR_HIGHER_OR_ZERO_SAFE:
939 		if (!cur || !new)
940 			break;
941 		fallthrough;
942 	case FTR_HIGHER_SAFE:
943 		ret = max(new, cur);
944 		break;
945 	default:
946 		BUG();
947 	}
948 
949 	return ret;
950 }
951 
sort_ftr_regs(void)952 static void __init sort_ftr_regs(void)
953 {
954 	unsigned int i;
955 
956 	for (i = 0; i < ARRAY_SIZE(arm64_ftr_regs); i++) {
957 		const struct arm64_ftr_reg *ftr_reg = arm64_ftr_regs[i].reg;
958 		const struct arm64_ftr_bits *ftr_bits = ftr_reg->ftr_bits;
959 		unsigned int j = 0;
960 
961 		/*
962 		 * Features here must be sorted in descending order with respect
963 		 * to their shift values and should not overlap with each other.
964 		 */
965 		for (; ftr_bits->width != 0; ftr_bits++, j++) {
966 			unsigned int width = ftr_reg->ftr_bits[j].width;
967 			unsigned int shift = ftr_reg->ftr_bits[j].shift;
968 			unsigned int prev_shift;
969 
970 			WARN((shift  + width) > 64,
971 				"%s has invalid feature at shift %d\n",
972 				ftr_reg->name, shift);
973 
974 			/*
975 			 * Skip the first feature. There is nothing to
976 			 * compare against for now.
977 			 */
978 			if (j == 0)
979 				continue;
980 
981 			prev_shift = ftr_reg->ftr_bits[j - 1].shift;
982 			WARN((shift + width) > prev_shift,
983 				"%s has feature overlap at shift %d\n",
984 				ftr_reg->name, shift);
985 		}
986 
987 		/*
988 		 * Skip the first register. There is nothing to
989 		 * compare against for now.
990 		 */
991 		if (i == 0)
992 			continue;
993 		/*
994 		 * Registers here must be sorted in ascending order with respect
995 		 * to sys_id for subsequent binary search in get_arm64_ftr_reg()
996 		 * to work correctly.
997 		 */
998 		BUG_ON(arm64_ftr_regs[i].sys_id <= arm64_ftr_regs[i - 1].sys_id);
999 	}
1000 }
1001 
1002 /*
1003  * Initialise the CPU feature register from Boot CPU values.
1004  * Also initiliases the strict_mask for the register.
1005  * Any bits that are not covered by an arm64_ftr_bits entry are considered
1006  * RES0 for the system-wide value, and must strictly match.
1007  */
init_cpu_ftr_reg(u32 sys_reg,u64 new)1008 static void init_cpu_ftr_reg(u32 sys_reg, u64 new)
1009 {
1010 	u64 val = 0;
1011 	u64 strict_mask = ~0x0ULL;
1012 	u64 user_mask = 0;
1013 	u64 valid_mask = 0;
1014 
1015 	const struct arm64_ftr_bits *ftrp;
1016 	struct arm64_ftr_reg *reg = get_arm64_ftr_reg(sys_reg);
1017 
1018 	if (!reg)
1019 		return;
1020 
1021 	for (ftrp = reg->ftr_bits; ftrp->width; ftrp++) {
1022 		u64 ftr_mask = arm64_ftr_mask(ftrp);
1023 		s64 ftr_new = arm64_ftr_value(ftrp, new);
1024 		s64 ftr_ovr = arm64_ftr_value(ftrp, reg->override->val);
1025 
1026 		if ((ftr_mask & reg->override->mask) == ftr_mask) {
1027 			s64 tmp = arm64_ftr_safe_value(ftrp, ftr_ovr, ftr_new);
1028 			char *str = NULL;
1029 
1030 			if (ftr_ovr != tmp) {
1031 				/* Unsafe, remove the override */
1032 				reg->override->mask &= ~ftr_mask;
1033 				reg->override->val &= ~ftr_mask;
1034 				tmp = ftr_ovr;
1035 				str = "ignoring override";
1036 			} else if (ftr_new != tmp) {
1037 				/* Override was valid */
1038 				ftr_new = tmp;
1039 				str = "forced";
1040 			} else {
1041 				/* Override was the safe value */
1042 				str = "already set";
1043 			}
1044 
1045 			pr_warn("%s[%d:%d]: %s to %llx\n",
1046 				reg->name,
1047 				ftrp->shift + ftrp->width - 1,
1048 				ftrp->shift, str,
1049 				tmp & (BIT(ftrp->width) - 1));
1050 		} else if ((ftr_mask & reg->override->val) == ftr_mask) {
1051 			reg->override->val &= ~ftr_mask;
1052 			pr_warn("%s[%d:%d]: impossible override, ignored\n",
1053 				reg->name,
1054 				ftrp->shift + ftrp->width - 1,
1055 				ftrp->shift);
1056 		}
1057 
1058 		val = arm64_ftr_set_value(ftrp, val, ftr_new);
1059 
1060 		valid_mask |= ftr_mask;
1061 		if (!ftrp->strict)
1062 			strict_mask &= ~ftr_mask;
1063 		if (ftrp->visible)
1064 			user_mask |= ftr_mask;
1065 		else
1066 			reg->user_val = arm64_ftr_set_value(ftrp,
1067 							    reg->user_val,
1068 							    ftrp->safe_val);
1069 	}
1070 
1071 	val &= valid_mask;
1072 
1073 	reg->sys_val = val;
1074 	reg->strict_mask = strict_mask;
1075 	reg->user_mask = user_mask;
1076 }
1077 
1078 extern const struct arm64_cpu_capabilities arm64_errata[];
1079 static const struct arm64_cpu_capabilities arm64_features[];
1080 
1081 static void __init
init_cpucap_indirect_list_from_array(const struct arm64_cpu_capabilities * caps)1082 init_cpucap_indirect_list_from_array(const struct arm64_cpu_capabilities *caps)
1083 {
1084 	for (; caps->matches; caps++) {
1085 		if (WARN(caps->capability >= ARM64_NCAPS,
1086 			"Invalid capability %d\n", caps->capability))
1087 			continue;
1088 		if (WARN(cpucap_ptrs[caps->capability],
1089 			"Duplicate entry for capability %d\n",
1090 			caps->capability))
1091 			continue;
1092 		cpucap_ptrs[caps->capability] = caps;
1093 	}
1094 }
1095 
init_cpucap_indirect_list(void)1096 static void __init init_cpucap_indirect_list(void)
1097 {
1098 	init_cpucap_indirect_list_from_array(arm64_features);
1099 	init_cpucap_indirect_list_from_array(arm64_errata);
1100 }
1101 
1102 static void __init setup_boot_cpu_capabilities(void);
1103 
init_32bit_cpu_features(struct cpuinfo_32bit * info)1104 static void init_32bit_cpu_features(struct cpuinfo_32bit *info)
1105 {
1106 	init_cpu_ftr_reg(SYS_ID_DFR0_EL1, info->reg_id_dfr0);
1107 	init_cpu_ftr_reg(SYS_ID_DFR1_EL1, info->reg_id_dfr1);
1108 	init_cpu_ftr_reg(SYS_ID_ISAR0_EL1, info->reg_id_isar0);
1109 	init_cpu_ftr_reg(SYS_ID_ISAR1_EL1, info->reg_id_isar1);
1110 	init_cpu_ftr_reg(SYS_ID_ISAR2_EL1, info->reg_id_isar2);
1111 	init_cpu_ftr_reg(SYS_ID_ISAR3_EL1, info->reg_id_isar3);
1112 	init_cpu_ftr_reg(SYS_ID_ISAR4_EL1, info->reg_id_isar4);
1113 	init_cpu_ftr_reg(SYS_ID_ISAR5_EL1, info->reg_id_isar5);
1114 	init_cpu_ftr_reg(SYS_ID_ISAR6_EL1, info->reg_id_isar6);
1115 	init_cpu_ftr_reg(SYS_ID_MMFR0_EL1, info->reg_id_mmfr0);
1116 	init_cpu_ftr_reg(SYS_ID_MMFR1_EL1, info->reg_id_mmfr1);
1117 	init_cpu_ftr_reg(SYS_ID_MMFR2_EL1, info->reg_id_mmfr2);
1118 	init_cpu_ftr_reg(SYS_ID_MMFR3_EL1, info->reg_id_mmfr3);
1119 	init_cpu_ftr_reg(SYS_ID_MMFR4_EL1, info->reg_id_mmfr4);
1120 	init_cpu_ftr_reg(SYS_ID_MMFR5_EL1, info->reg_id_mmfr5);
1121 	init_cpu_ftr_reg(SYS_ID_PFR0_EL1, info->reg_id_pfr0);
1122 	init_cpu_ftr_reg(SYS_ID_PFR1_EL1, info->reg_id_pfr1);
1123 	init_cpu_ftr_reg(SYS_ID_PFR2_EL1, info->reg_id_pfr2);
1124 	init_cpu_ftr_reg(SYS_MVFR0_EL1, info->reg_mvfr0);
1125 	init_cpu_ftr_reg(SYS_MVFR1_EL1, info->reg_mvfr1);
1126 	init_cpu_ftr_reg(SYS_MVFR2_EL1, info->reg_mvfr2);
1127 }
1128 
1129 #ifdef CONFIG_ARM64_PSEUDO_NMI
1130 static bool enable_pseudo_nmi;
1131 
early_enable_pseudo_nmi(char * p)1132 static int __init early_enable_pseudo_nmi(char *p)
1133 {
1134 	return kstrtobool(p, &enable_pseudo_nmi);
1135 }
1136 early_param("irqchip.gicv3_pseudo_nmi", early_enable_pseudo_nmi);
1137 
detect_system_supports_pseudo_nmi(void)1138 static __init void detect_system_supports_pseudo_nmi(void)
1139 {
1140 	struct device_node *np;
1141 
1142 	if (!enable_pseudo_nmi)
1143 		return;
1144 
1145 	/*
1146 	 * Detect broken MediaTek firmware that doesn't properly save and
1147 	 * restore GIC priorities.
1148 	 */
1149 	np = of_find_compatible_node(NULL, NULL, "arm,gic-v3");
1150 	if (np && of_property_read_bool(np, "mediatek,broken-save-restore-fw")) {
1151 		pr_info("Pseudo-NMI disabled due to MediaTek Chromebook GICR save problem\n");
1152 		enable_pseudo_nmi = false;
1153 	}
1154 	of_node_put(np);
1155 }
1156 #else /* CONFIG_ARM64_PSEUDO_NMI */
detect_system_supports_pseudo_nmi(void)1157 static inline void detect_system_supports_pseudo_nmi(void) { }
1158 #endif
1159 
init_cpu_features(struct cpuinfo_arm64 * info)1160 void __init init_cpu_features(struct cpuinfo_arm64 *info)
1161 {
1162 	/* Before we start using the tables, make sure it is sorted */
1163 	sort_ftr_regs();
1164 
1165 	init_cpu_ftr_reg(SYS_CTR_EL0, info->reg_ctr);
1166 	init_cpu_ftr_reg(SYS_DCZID_EL0, info->reg_dczid);
1167 	init_cpu_ftr_reg(SYS_CNTFRQ_EL0, info->reg_cntfrq);
1168 	init_cpu_ftr_reg(SYS_ID_AA64DFR0_EL1, info->reg_id_aa64dfr0);
1169 	init_cpu_ftr_reg(SYS_ID_AA64DFR1_EL1, info->reg_id_aa64dfr1);
1170 	init_cpu_ftr_reg(SYS_ID_AA64ISAR0_EL1, info->reg_id_aa64isar0);
1171 	init_cpu_ftr_reg(SYS_ID_AA64ISAR1_EL1, info->reg_id_aa64isar1);
1172 	init_cpu_ftr_reg(SYS_ID_AA64ISAR2_EL1, info->reg_id_aa64isar2);
1173 	init_cpu_ftr_reg(SYS_ID_AA64ISAR3_EL1, info->reg_id_aa64isar3);
1174 	init_cpu_ftr_reg(SYS_ID_AA64MMFR0_EL1, info->reg_id_aa64mmfr0);
1175 	init_cpu_ftr_reg(SYS_ID_AA64MMFR1_EL1, info->reg_id_aa64mmfr1);
1176 	init_cpu_ftr_reg(SYS_ID_AA64MMFR2_EL1, info->reg_id_aa64mmfr2);
1177 	init_cpu_ftr_reg(SYS_ID_AA64MMFR3_EL1, info->reg_id_aa64mmfr3);
1178 	init_cpu_ftr_reg(SYS_ID_AA64MMFR4_EL1, info->reg_id_aa64mmfr4);
1179 	init_cpu_ftr_reg(SYS_ID_AA64PFR0_EL1, info->reg_id_aa64pfr0);
1180 	init_cpu_ftr_reg(SYS_ID_AA64PFR1_EL1, info->reg_id_aa64pfr1);
1181 	init_cpu_ftr_reg(SYS_ID_AA64PFR2_EL1, info->reg_id_aa64pfr2);
1182 	init_cpu_ftr_reg(SYS_ID_AA64ZFR0_EL1, info->reg_id_aa64zfr0);
1183 	init_cpu_ftr_reg(SYS_ID_AA64SMFR0_EL1, info->reg_id_aa64smfr0);
1184 	init_cpu_ftr_reg(SYS_ID_AA64FPFR0_EL1, info->reg_id_aa64fpfr0);
1185 
1186 	if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0))
1187 		init_32bit_cpu_features(&info->aarch32);
1188 
1189 	if (IS_ENABLED(CONFIG_ARM64_SVE) &&
1190 	    id_aa64pfr0_sve(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1))) {
1191 		unsigned long cpacr = cpacr_save_enable_kernel_sve();
1192 
1193 		vec_init_vq_map(ARM64_VEC_SVE);
1194 
1195 		cpacr_restore(cpacr);
1196 	}
1197 
1198 	if (IS_ENABLED(CONFIG_ARM64_SME) &&
1199 	    id_aa64pfr1_sme(read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1))) {
1200 		unsigned long cpacr = cpacr_save_enable_kernel_sme();
1201 
1202 		vec_init_vq_map(ARM64_VEC_SME);
1203 
1204 		cpacr_restore(cpacr);
1205 	}
1206 
1207 	if (id_aa64pfr0_mpam(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1))) {
1208 		info->reg_mpamidr = read_cpuid(MPAMIDR_EL1);
1209 		init_cpu_ftr_reg(SYS_MPAMIDR_EL1, info->reg_mpamidr);
1210 	}
1211 
1212 	if (id_aa64pfr1_mte(info->reg_id_aa64pfr1))
1213 		init_cpu_ftr_reg(SYS_GMID_EL1, info->reg_gmid);
1214 }
1215 
update_cpu_ftr_reg(struct arm64_ftr_reg * reg,u64 new)1216 static void update_cpu_ftr_reg(struct arm64_ftr_reg *reg, u64 new)
1217 {
1218 	const struct arm64_ftr_bits *ftrp;
1219 
1220 	for (ftrp = reg->ftr_bits; ftrp->width; ftrp++) {
1221 		s64 ftr_cur = arm64_ftr_value(ftrp, reg->sys_val);
1222 		s64 ftr_new = arm64_ftr_value(ftrp, new);
1223 
1224 		if (ftr_cur == ftr_new)
1225 			continue;
1226 		/* Find a safe value */
1227 		ftr_new = arm64_ftr_safe_value(ftrp, ftr_new, ftr_cur);
1228 		reg->sys_val = arm64_ftr_set_value(ftrp, reg->sys_val, ftr_new);
1229 	}
1230 
1231 }
1232 
check_update_ftr_reg(u32 sys_id,int cpu,u64 val,u64 boot)1233 static int check_update_ftr_reg(u32 sys_id, int cpu, u64 val, u64 boot)
1234 {
1235 	struct arm64_ftr_reg *regp = get_arm64_ftr_reg(sys_id);
1236 
1237 	if (!regp)
1238 		return 0;
1239 
1240 	update_cpu_ftr_reg(regp, val);
1241 	if ((boot & regp->strict_mask) == (val & regp->strict_mask))
1242 		return 0;
1243 	pr_warn("SANITY CHECK: Unexpected variation in %s. Boot CPU: %#016llx, CPU%d: %#016llx\n",
1244 			regp->name, boot, cpu, val);
1245 	return 1;
1246 }
1247 
relax_cpu_ftr_reg(u32 sys_id,int field)1248 static void relax_cpu_ftr_reg(u32 sys_id, int field)
1249 {
1250 	const struct arm64_ftr_bits *ftrp;
1251 	struct arm64_ftr_reg *regp = get_arm64_ftr_reg(sys_id);
1252 
1253 	if (!regp)
1254 		return;
1255 
1256 	for (ftrp = regp->ftr_bits; ftrp->width; ftrp++) {
1257 		if (ftrp->shift == field) {
1258 			regp->strict_mask &= ~arm64_ftr_mask(ftrp);
1259 			break;
1260 		}
1261 	}
1262 
1263 	/* Bogus field? */
1264 	WARN_ON(!ftrp->width);
1265 }
1266 
lazy_init_32bit_cpu_features(struct cpuinfo_arm64 * info,struct cpuinfo_arm64 * boot)1267 static void lazy_init_32bit_cpu_features(struct cpuinfo_arm64 *info,
1268 					 struct cpuinfo_arm64 *boot)
1269 {
1270 	static bool boot_cpu_32bit_regs_overridden = false;
1271 
1272 	if (!allow_mismatched_32bit_el0 || boot_cpu_32bit_regs_overridden)
1273 		return;
1274 
1275 	if (id_aa64pfr0_32bit_el0(boot->reg_id_aa64pfr0))
1276 		return;
1277 
1278 	boot->aarch32 = info->aarch32;
1279 	init_32bit_cpu_features(&boot->aarch32);
1280 	boot_cpu_32bit_regs_overridden = true;
1281 }
1282 
update_32bit_cpu_features(int cpu,struct cpuinfo_32bit * info,struct cpuinfo_32bit * boot)1283 static int update_32bit_cpu_features(int cpu, struct cpuinfo_32bit *info,
1284 				     struct cpuinfo_32bit *boot)
1285 {
1286 	int taint = 0;
1287 	u64 pfr0 = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
1288 
1289 	/*
1290 	 * If we don't have AArch32 at EL1, then relax the strictness of
1291 	 * EL1-dependent register fields to avoid spurious sanity check fails.
1292 	 */
1293 	if (!id_aa64pfr0_32bit_el1(pfr0)) {
1294 		relax_cpu_ftr_reg(SYS_ID_ISAR4_EL1, ID_ISAR4_EL1_SMC_SHIFT);
1295 		relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_Virt_frac_SHIFT);
1296 		relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_Sec_frac_SHIFT);
1297 		relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_Virtualization_SHIFT);
1298 		relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_Security_SHIFT);
1299 		relax_cpu_ftr_reg(SYS_ID_PFR1_EL1, ID_PFR1_EL1_ProgMod_SHIFT);
1300 	}
1301 
1302 	taint |= check_update_ftr_reg(SYS_ID_DFR0_EL1, cpu,
1303 				      info->reg_id_dfr0, boot->reg_id_dfr0);
1304 	taint |= check_update_ftr_reg(SYS_ID_DFR1_EL1, cpu,
1305 				      info->reg_id_dfr1, boot->reg_id_dfr1);
1306 	taint |= check_update_ftr_reg(SYS_ID_ISAR0_EL1, cpu,
1307 				      info->reg_id_isar0, boot->reg_id_isar0);
1308 	taint |= check_update_ftr_reg(SYS_ID_ISAR1_EL1, cpu,
1309 				      info->reg_id_isar1, boot->reg_id_isar1);
1310 	taint |= check_update_ftr_reg(SYS_ID_ISAR2_EL1, cpu,
1311 				      info->reg_id_isar2, boot->reg_id_isar2);
1312 	taint |= check_update_ftr_reg(SYS_ID_ISAR3_EL1, cpu,
1313 				      info->reg_id_isar3, boot->reg_id_isar3);
1314 	taint |= check_update_ftr_reg(SYS_ID_ISAR4_EL1, cpu,
1315 				      info->reg_id_isar4, boot->reg_id_isar4);
1316 	taint |= check_update_ftr_reg(SYS_ID_ISAR5_EL1, cpu,
1317 				      info->reg_id_isar5, boot->reg_id_isar5);
1318 	taint |= check_update_ftr_reg(SYS_ID_ISAR6_EL1, cpu,
1319 				      info->reg_id_isar6, boot->reg_id_isar6);
1320 
1321 	/*
1322 	 * Regardless of the value of the AuxReg field, the AIFSR, ADFSR, and
1323 	 * ACTLR formats could differ across CPUs and therefore would have to
1324 	 * be trapped for virtualization anyway.
1325 	 */
1326 	taint |= check_update_ftr_reg(SYS_ID_MMFR0_EL1, cpu,
1327 				      info->reg_id_mmfr0, boot->reg_id_mmfr0);
1328 	taint |= check_update_ftr_reg(SYS_ID_MMFR1_EL1, cpu,
1329 				      info->reg_id_mmfr1, boot->reg_id_mmfr1);
1330 	taint |= check_update_ftr_reg(SYS_ID_MMFR2_EL1, cpu,
1331 				      info->reg_id_mmfr2, boot->reg_id_mmfr2);
1332 	taint |= check_update_ftr_reg(SYS_ID_MMFR3_EL1, cpu,
1333 				      info->reg_id_mmfr3, boot->reg_id_mmfr3);
1334 	taint |= check_update_ftr_reg(SYS_ID_MMFR4_EL1, cpu,
1335 				      info->reg_id_mmfr4, boot->reg_id_mmfr4);
1336 	taint |= check_update_ftr_reg(SYS_ID_MMFR5_EL1, cpu,
1337 				      info->reg_id_mmfr5, boot->reg_id_mmfr5);
1338 	taint |= check_update_ftr_reg(SYS_ID_PFR0_EL1, cpu,
1339 				      info->reg_id_pfr0, boot->reg_id_pfr0);
1340 	taint |= check_update_ftr_reg(SYS_ID_PFR1_EL1, cpu,
1341 				      info->reg_id_pfr1, boot->reg_id_pfr1);
1342 	taint |= check_update_ftr_reg(SYS_ID_PFR2_EL1, cpu,
1343 				      info->reg_id_pfr2, boot->reg_id_pfr2);
1344 	taint |= check_update_ftr_reg(SYS_MVFR0_EL1, cpu,
1345 				      info->reg_mvfr0, boot->reg_mvfr0);
1346 	taint |= check_update_ftr_reg(SYS_MVFR1_EL1, cpu,
1347 				      info->reg_mvfr1, boot->reg_mvfr1);
1348 	taint |= check_update_ftr_reg(SYS_MVFR2_EL1, cpu,
1349 				      info->reg_mvfr2, boot->reg_mvfr2);
1350 
1351 	return taint;
1352 }
1353 
1354 /*
1355  * Update system wide CPU feature registers with the values from a
1356  * non-boot CPU. Also performs SANITY checks to make sure that there
1357  * aren't any insane variations from that of the boot CPU.
1358  */
update_cpu_features(int cpu,struct cpuinfo_arm64 * info,struct cpuinfo_arm64 * boot)1359 void update_cpu_features(int cpu,
1360 			 struct cpuinfo_arm64 *info,
1361 			 struct cpuinfo_arm64 *boot)
1362 {
1363 	int taint = 0;
1364 
1365 	/*
1366 	 * The kernel can handle differing I-cache policies, but otherwise
1367 	 * caches should look identical. Userspace JITs will make use of
1368 	 * *minLine.
1369 	 */
1370 	taint |= check_update_ftr_reg(SYS_CTR_EL0, cpu,
1371 				      info->reg_ctr, boot->reg_ctr);
1372 
1373 	/*
1374 	 * Userspace may perform DC ZVA instructions. Mismatched block sizes
1375 	 * could result in too much or too little memory being zeroed if a
1376 	 * process is preempted and migrated between CPUs.
1377 	 */
1378 	taint |= check_update_ftr_reg(SYS_DCZID_EL0, cpu,
1379 				      info->reg_dczid, boot->reg_dczid);
1380 
1381 	/* If different, timekeeping will be broken (especially with KVM) */
1382 	taint |= check_update_ftr_reg(SYS_CNTFRQ_EL0, cpu,
1383 				      info->reg_cntfrq, boot->reg_cntfrq);
1384 
1385 	/*
1386 	 * The kernel uses self-hosted debug features and expects CPUs to
1387 	 * support identical debug features. We presently need CTX_CMPs, WRPs,
1388 	 * and BRPs to be identical.
1389 	 * ID_AA64DFR1 is currently RES0.
1390 	 */
1391 	taint |= check_update_ftr_reg(SYS_ID_AA64DFR0_EL1, cpu,
1392 				      info->reg_id_aa64dfr0, boot->reg_id_aa64dfr0);
1393 	taint |= check_update_ftr_reg(SYS_ID_AA64DFR1_EL1, cpu,
1394 				      info->reg_id_aa64dfr1, boot->reg_id_aa64dfr1);
1395 	/*
1396 	 * Even in big.LITTLE, processors should be identical instruction-set
1397 	 * wise.
1398 	 */
1399 	taint |= check_update_ftr_reg(SYS_ID_AA64ISAR0_EL1, cpu,
1400 				      info->reg_id_aa64isar0, boot->reg_id_aa64isar0);
1401 	taint |= check_update_ftr_reg(SYS_ID_AA64ISAR1_EL1, cpu,
1402 				      info->reg_id_aa64isar1, boot->reg_id_aa64isar1);
1403 	taint |= check_update_ftr_reg(SYS_ID_AA64ISAR2_EL1, cpu,
1404 				      info->reg_id_aa64isar2, boot->reg_id_aa64isar2);
1405 	taint |= check_update_ftr_reg(SYS_ID_AA64ISAR3_EL1, cpu,
1406 				      info->reg_id_aa64isar3, boot->reg_id_aa64isar3);
1407 
1408 	/*
1409 	 * Differing PARange support is fine as long as all peripherals and
1410 	 * memory are mapped within the minimum PARange of all CPUs.
1411 	 * Linux should not care about secure memory.
1412 	 */
1413 	taint |= check_update_ftr_reg(SYS_ID_AA64MMFR0_EL1, cpu,
1414 				      info->reg_id_aa64mmfr0, boot->reg_id_aa64mmfr0);
1415 	taint |= check_update_ftr_reg(SYS_ID_AA64MMFR1_EL1, cpu,
1416 				      info->reg_id_aa64mmfr1, boot->reg_id_aa64mmfr1);
1417 	taint |= check_update_ftr_reg(SYS_ID_AA64MMFR2_EL1, cpu,
1418 				      info->reg_id_aa64mmfr2, boot->reg_id_aa64mmfr2);
1419 	taint |= check_update_ftr_reg(SYS_ID_AA64MMFR3_EL1, cpu,
1420 				      info->reg_id_aa64mmfr3, boot->reg_id_aa64mmfr3);
1421 	taint |= check_update_ftr_reg(SYS_ID_AA64MMFR4_EL1, cpu,
1422 				      info->reg_id_aa64mmfr4, boot->reg_id_aa64mmfr4);
1423 
1424 	taint |= check_update_ftr_reg(SYS_ID_AA64PFR0_EL1, cpu,
1425 				      info->reg_id_aa64pfr0, boot->reg_id_aa64pfr0);
1426 	taint |= check_update_ftr_reg(SYS_ID_AA64PFR1_EL1, cpu,
1427 				      info->reg_id_aa64pfr1, boot->reg_id_aa64pfr1);
1428 	taint |= check_update_ftr_reg(SYS_ID_AA64PFR2_EL1, cpu,
1429 				      info->reg_id_aa64pfr2, boot->reg_id_aa64pfr2);
1430 
1431 	taint |= check_update_ftr_reg(SYS_ID_AA64ZFR0_EL1, cpu,
1432 				      info->reg_id_aa64zfr0, boot->reg_id_aa64zfr0);
1433 
1434 	taint |= check_update_ftr_reg(SYS_ID_AA64SMFR0_EL1, cpu,
1435 				      info->reg_id_aa64smfr0, boot->reg_id_aa64smfr0);
1436 
1437 	taint |= check_update_ftr_reg(SYS_ID_AA64FPFR0_EL1, cpu,
1438 				      info->reg_id_aa64fpfr0, boot->reg_id_aa64fpfr0);
1439 
1440 	/* Probe vector lengths */
1441 	if (IS_ENABLED(CONFIG_ARM64_SVE) &&
1442 	    id_aa64pfr0_sve(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1))) {
1443 		if (!system_capabilities_finalized()) {
1444 			unsigned long cpacr = cpacr_save_enable_kernel_sve();
1445 
1446 			vec_update_vq_map(ARM64_VEC_SVE);
1447 
1448 			cpacr_restore(cpacr);
1449 		}
1450 	}
1451 
1452 	if (IS_ENABLED(CONFIG_ARM64_SME) &&
1453 	    id_aa64pfr1_sme(read_sanitised_ftr_reg(SYS_ID_AA64PFR1_EL1))) {
1454 		unsigned long cpacr = cpacr_save_enable_kernel_sme();
1455 
1456 		/* Probe vector lengths */
1457 		if (!system_capabilities_finalized())
1458 			vec_update_vq_map(ARM64_VEC_SME);
1459 
1460 		cpacr_restore(cpacr);
1461 	}
1462 
1463 	if (id_aa64pfr0_mpam(read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1))) {
1464 		info->reg_mpamidr = read_cpuid(MPAMIDR_EL1);
1465 		taint |= check_update_ftr_reg(SYS_MPAMIDR_EL1, cpu,
1466 					info->reg_mpamidr, boot->reg_mpamidr);
1467 	}
1468 
1469 	/*
1470 	 * The kernel uses the LDGM/STGM instructions and the number of tags
1471 	 * they read/write depends on the GMID_EL1.BS field. Check that the
1472 	 * value is the same on all CPUs.
1473 	 */
1474 	if (IS_ENABLED(CONFIG_ARM64_MTE) &&
1475 	    id_aa64pfr1_mte(info->reg_id_aa64pfr1)) {
1476 		taint |= check_update_ftr_reg(SYS_GMID_EL1, cpu,
1477 					      info->reg_gmid, boot->reg_gmid);
1478 	}
1479 
1480 	/*
1481 	 * If we don't have AArch32 at all then skip the checks entirely
1482 	 * as the register values may be UNKNOWN and we're not going to be
1483 	 * using them for anything.
1484 	 *
1485 	 * This relies on a sanitised view of the AArch64 ID registers
1486 	 * (e.g. SYS_ID_AA64PFR0_EL1), so we call it last.
1487 	 */
1488 	if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) {
1489 		lazy_init_32bit_cpu_features(info, boot);
1490 		taint |= update_32bit_cpu_features(cpu, &info->aarch32,
1491 						   &boot->aarch32);
1492 	}
1493 
1494 	/*
1495 	 * Mismatched CPU features are a recipe for disaster. Don't even
1496 	 * pretend to support them.
1497 	 */
1498 	if (taint) {
1499 		pr_warn_once("Unsupported CPU feature variation detected.\n");
1500 		add_taint(TAINT_CPU_OUT_OF_SPEC, LOCKDEP_STILL_OK);
1501 	}
1502 }
1503 
read_sanitised_ftr_reg(u32 id)1504 u64 read_sanitised_ftr_reg(u32 id)
1505 {
1506 	struct arm64_ftr_reg *regp = get_arm64_ftr_reg(id);
1507 
1508 	if (!regp)
1509 		return 0;
1510 	return regp->sys_val;
1511 }
1512 EXPORT_SYMBOL_GPL(read_sanitised_ftr_reg);
1513 
1514 #define read_sysreg_case(r)	\
1515 	case r:		val = read_sysreg_s(r); break;
1516 
1517 /*
1518  * __read_sysreg_by_encoding() - Used by a STARTING cpu before cpuinfo is populated.
1519  * Read the system register on the current CPU
1520  */
__read_sysreg_by_encoding(u32 sys_id)1521 u64 __read_sysreg_by_encoding(u32 sys_id)
1522 {
1523 	struct arm64_ftr_reg *regp;
1524 	u64 val;
1525 
1526 	switch (sys_id) {
1527 	read_sysreg_case(SYS_ID_PFR0_EL1);
1528 	read_sysreg_case(SYS_ID_PFR1_EL1);
1529 	read_sysreg_case(SYS_ID_PFR2_EL1);
1530 	read_sysreg_case(SYS_ID_DFR0_EL1);
1531 	read_sysreg_case(SYS_ID_DFR1_EL1);
1532 	read_sysreg_case(SYS_ID_MMFR0_EL1);
1533 	read_sysreg_case(SYS_ID_MMFR1_EL1);
1534 	read_sysreg_case(SYS_ID_MMFR2_EL1);
1535 	read_sysreg_case(SYS_ID_MMFR3_EL1);
1536 	read_sysreg_case(SYS_ID_MMFR4_EL1);
1537 	read_sysreg_case(SYS_ID_MMFR5_EL1);
1538 	read_sysreg_case(SYS_ID_ISAR0_EL1);
1539 	read_sysreg_case(SYS_ID_ISAR1_EL1);
1540 	read_sysreg_case(SYS_ID_ISAR2_EL1);
1541 	read_sysreg_case(SYS_ID_ISAR3_EL1);
1542 	read_sysreg_case(SYS_ID_ISAR4_EL1);
1543 	read_sysreg_case(SYS_ID_ISAR5_EL1);
1544 	read_sysreg_case(SYS_ID_ISAR6_EL1);
1545 	read_sysreg_case(SYS_MVFR0_EL1);
1546 	read_sysreg_case(SYS_MVFR1_EL1);
1547 	read_sysreg_case(SYS_MVFR2_EL1);
1548 
1549 	read_sysreg_case(SYS_ID_AA64PFR0_EL1);
1550 	read_sysreg_case(SYS_ID_AA64PFR1_EL1);
1551 	read_sysreg_case(SYS_ID_AA64PFR2_EL1);
1552 	read_sysreg_case(SYS_ID_AA64ZFR0_EL1);
1553 	read_sysreg_case(SYS_ID_AA64SMFR0_EL1);
1554 	read_sysreg_case(SYS_ID_AA64FPFR0_EL1);
1555 	read_sysreg_case(SYS_ID_AA64DFR0_EL1);
1556 	read_sysreg_case(SYS_ID_AA64DFR1_EL1);
1557 	read_sysreg_case(SYS_ID_AA64MMFR0_EL1);
1558 	read_sysreg_case(SYS_ID_AA64MMFR1_EL1);
1559 	read_sysreg_case(SYS_ID_AA64MMFR2_EL1);
1560 	read_sysreg_case(SYS_ID_AA64MMFR3_EL1);
1561 	read_sysreg_case(SYS_ID_AA64MMFR4_EL1);
1562 	read_sysreg_case(SYS_ID_AA64ISAR0_EL1);
1563 	read_sysreg_case(SYS_ID_AA64ISAR1_EL1);
1564 	read_sysreg_case(SYS_ID_AA64ISAR2_EL1);
1565 	read_sysreg_case(SYS_ID_AA64ISAR3_EL1);
1566 
1567 	read_sysreg_case(SYS_CNTFRQ_EL0);
1568 	read_sysreg_case(SYS_CTR_EL0);
1569 	read_sysreg_case(SYS_DCZID_EL0);
1570 
1571 	default:
1572 		BUG();
1573 		return 0;
1574 	}
1575 
1576 	regp  = get_arm64_ftr_reg(sys_id);
1577 	if (regp) {
1578 		val &= ~regp->override->mask;
1579 		val |= (regp->override->val & regp->override->mask);
1580 	}
1581 
1582 	return val;
1583 }
1584 
1585 #include <linux/irqchip/arm-gic-v3.h>
1586 
1587 static bool
has_always(const struct arm64_cpu_capabilities * entry,int scope)1588 has_always(const struct arm64_cpu_capabilities *entry, int scope)
1589 {
1590 	return true;
1591 }
1592 
1593 static bool
feature_matches(u64 reg,const struct arm64_cpu_capabilities * entry)1594 feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry)
1595 {
1596 	int val, min, max;
1597 	u64 tmp;
1598 
1599 	val = cpuid_feature_extract_field_width(reg, entry->field_pos,
1600 						entry->field_width,
1601 						entry->sign);
1602 
1603 	tmp = entry->min_field_value;
1604 	tmp <<= entry->field_pos;
1605 
1606 	min = cpuid_feature_extract_field_width(tmp, entry->field_pos,
1607 						entry->field_width,
1608 						entry->sign);
1609 
1610 	tmp = entry->max_field_value;
1611 	tmp <<= entry->field_pos;
1612 
1613 	max = cpuid_feature_extract_field_width(tmp, entry->field_pos,
1614 						entry->field_width,
1615 						entry->sign);
1616 
1617 	return val >= min && val <= max;
1618 }
1619 
1620 static u64
read_scoped_sysreg(const struct arm64_cpu_capabilities * entry,int scope)1621 read_scoped_sysreg(const struct arm64_cpu_capabilities *entry, int scope)
1622 {
1623 	WARN_ON(scope == SCOPE_LOCAL_CPU && preemptible());
1624 	if (scope == SCOPE_SYSTEM)
1625 		return read_sanitised_ftr_reg(entry->sys_reg);
1626 	else
1627 		return __read_sysreg_by_encoding(entry->sys_reg);
1628 }
1629 
1630 static bool
has_user_cpuid_feature(const struct arm64_cpu_capabilities * entry,int scope)1631 has_user_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope)
1632 {
1633 	int mask;
1634 	struct arm64_ftr_reg *regp;
1635 	u64 val = read_scoped_sysreg(entry, scope);
1636 
1637 	regp = get_arm64_ftr_reg(entry->sys_reg);
1638 	if (!regp)
1639 		return false;
1640 
1641 	mask = cpuid_feature_extract_unsigned_field_width(regp->user_mask,
1642 							  entry->field_pos,
1643 							  entry->field_width);
1644 	if (!mask)
1645 		return false;
1646 
1647 	return feature_matches(val, entry);
1648 }
1649 
1650 static bool
has_cpuid_feature(const struct arm64_cpu_capabilities * entry,int scope)1651 has_cpuid_feature(const struct arm64_cpu_capabilities *entry, int scope)
1652 {
1653 	u64 val = read_scoped_sysreg(entry, scope);
1654 	return feature_matches(val, entry);
1655 }
1656 
system_32bit_el0_cpumask(void)1657 const struct cpumask *system_32bit_el0_cpumask(void)
1658 {
1659 	if (!system_supports_32bit_el0())
1660 		return cpu_none_mask;
1661 
1662 	if (static_branch_unlikely(&arm64_mismatched_32bit_el0))
1663 		return cpu_32bit_el0_mask;
1664 
1665 	return cpu_possible_mask;
1666 }
1667 
task_cpu_fallback_mask(struct task_struct * p)1668 const struct cpumask *task_cpu_fallback_mask(struct task_struct *p)
1669 {
1670 	return __task_cpu_possible_mask(p, housekeeping_cpumask(HK_TYPE_TICK));
1671 }
1672 
parse_32bit_el0_param(char * str)1673 static int __init parse_32bit_el0_param(char *str)
1674 {
1675 	allow_mismatched_32bit_el0 = true;
1676 	return 0;
1677 }
1678 early_param("allow_mismatched_32bit_el0", parse_32bit_el0_param);
1679 
aarch32_el0_show(struct device * dev,struct device_attribute * attr,char * buf)1680 static ssize_t aarch32_el0_show(struct device *dev,
1681 				struct device_attribute *attr, char *buf)
1682 {
1683 	const struct cpumask *mask = system_32bit_el0_cpumask();
1684 
1685 	return sysfs_emit(buf, "%*pbl\n", cpumask_pr_args(mask));
1686 }
1687 static const DEVICE_ATTR_RO(aarch32_el0);
1688 
aarch32_el0_sysfs_init(void)1689 static int __init aarch32_el0_sysfs_init(void)
1690 {
1691 	struct device *dev_root;
1692 	int ret = 0;
1693 
1694 	if (!allow_mismatched_32bit_el0)
1695 		return 0;
1696 
1697 	dev_root = bus_get_dev_root(&cpu_subsys);
1698 	if (dev_root) {
1699 		ret = device_create_file(dev_root, &dev_attr_aarch32_el0);
1700 		put_device(dev_root);
1701 	}
1702 	return ret;
1703 }
1704 device_initcall(aarch32_el0_sysfs_init);
1705 
has_32bit_el0(const struct arm64_cpu_capabilities * entry,int scope)1706 static bool has_32bit_el0(const struct arm64_cpu_capabilities *entry, int scope)
1707 {
1708 	if (!has_cpuid_feature(entry, scope))
1709 		return allow_mismatched_32bit_el0;
1710 
1711 	if (scope == SCOPE_SYSTEM)
1712 		pr_info("detected: 32-bit EL0 Support\n");
1713 
1714 	return true;
1715 }
1716 
has_useable_gicv3_cpuif(const struct arm64_cpu_capabilities * entry,int scope)1717 static bool has_useable_gicv3_cpuif(const struct arm64_cpu_capabilities *entry, int scope)
1718 {
1719 	bool has_sre;
1720 
1721 	if (!has_cpuid_feature(entry, scope))
1722 		return false;
1723 
1724 	has_sre = gic_enable_sre();
1725 	if (!has_sre)
1726 		pr_warn_once("%s present but disabled by higher exception level\n",
1727 			     entry->desc);
1728 
1729 	return has_sre;
1730 }
1731 
has_cache_idc(const struct arm64_cpu_capabilities * entry,int scope)1732 static bool has_cache_idc(const struct arm64_cpu_capabilities *entry,
1733 			  int scope)
1734 {
1735 	u64 ctr;
1736 
1737 	if (scope == SCOPE_SYSTEM)
1738 		ctr = arm64_ftr_reg_ctrel0.sys_val;
1739 	else
1740 		ctr = read_cpuid_effective_cachetype();
1741 
1742 	return ctr & BIT(CTR_EL0_IDC_SHIFT);
1743 }
1744 
cpu_emulate_effective_ctr(const struct arm64_cpu_capabilities * __unused)1745 static void cpu_emulate_effective_ctr(const struct arm64_cpu_capabilities *__unused)
1746 {
1747 	/*
1748 	 * If the CPU exposes raw CTR_EL0.IDC = 0, while effectively
1749 	 * CTR_EL0.IDC = 1 (from CLIDR values), we need to trap accesses
1750 	 * to the CTR_EL0 on this CPU and emulate it with the real/safe
1751 	 * value.
1752 	 */
1753 	if (!(read_cpuid_cachetype() & BIT(CTR_EL0_IDC_SHIFT)))
1754 		sysreg_clear_set(sctlr_el1, SCTLR_EL1_UCT, 0);
1755 }
1756 
has_cache_dic(const struct arm64_cpu_capabilities * entry,int scope)1757 static bool has_cache_dic(const struct arm64_cpu_capabilities *entry,
1758 			  int scope)
1759 {
1760 	u64 ctr;
1761 
1762 	if (scope == SCOPE_SYSTEM)
1763 		ctr = arm64_ftr_reg_ctrel0.sys_val;
1764 	else
1765 		ctr = read_cpuid_cachetype();
1766 
1767 	return ctr & BIT(CTR_EL0_DIC_SHIFT);
1768 }
1769 
1770 static bool __maybe_unused
has_useable_cnp(const struct arm64_cpu_capabilities * entry,int scope)1771 has_useable_cnp(const struct arm64_cpu_capabilities *entry, int scope)
1772 {
1773 	/*
1774 	 * Kdump isn't guaranteed to power-off all secondary CPUs, CNP
1775 	 * may share TLB entries with a CPU stuck in the crashed
1776 	 * kernel.
1777 	 */
1778 	if (is_kdump_kernel())
1779 		return false;
1780 
1781 	if (cpus_have_cap(ARM64_WORKAROUND_NVIDIA_CARMEL_CNP))
1782 		return false;
1783 
1784 	return has_cpuid_feature(entry, scope);
1785 }
1786 
1787 static bool __meltdown_safe = true;
1788 static int __kpti_forced; /* 0: not forced, >0: forced on, <0: forced off */
1789 
unmap_kernel_at_el0(const struct arm64_cpu_capabilities * entry,int scope)1790 static bool unmap_kernel_at_el0(const struct arm64_cpu_capabilities *entry,
1791 				int scope)
1792 {
1793 	/* List of CPUs that are not vulnerable and don't need KPTI */
1794 	static const struct midr_range kpti_safe_list[] = {
1795 		MIDR_ALL_VERSIONS(MIDR_CAVIUM_THUNDERX2),
1796 		MIDR_ALL_VERSIONS(MIDR_BRCM_VULCAN),
1797 		MIDR_ALL_VERSIONS(MIDR_BRAHMA_B53),
1798 		MIDR_ALL_VERSIONS(MIDR_CORTEX_A35),
1799 		MIDR_ALL_VERSIONS(MIDR_CORTEX_A53),
1800 		MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
1801 		MIDR_ALL_VERSIONS(MIDR_CORTEX_A57),
1802 		MIDR_ALL_VERSIONS(MIDR_CORTEX_A72),
1803 		MIDR_ALL_VERSIONS(MIDR_CORTEX_A73),
1804 		MIDR_ALL_VERSIONS(MIDR_HISI_TSV110),
1805 		MIDR_ALL_VERSIONS(MIDR_NVIDIA_CARMEL),
1806 		MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_2XX_GOLD),
1807 		MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_2XX_SILVER),
1808 		MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_3XX_SILVER),
1809 		MIDR_ALL_VERSIONS(MIDR_QCOM_KRYO_4XX_SILVER),
1810 		{ /* sentinel */ }
1811 	};
1812 	char const *str = "kpti command line option";
1813 	bool meltdown_safe;
1814 
1815 	meltdown_safe = is_midr_in_range_list(kpti_safe_list);
1816 
1817 	/* Defer to CPU feature registers */
1818 	if (has_cpuid_feature(entry, scope))
1819 		meltdown_safe = true;
1820 
1821 	if (!meltdown_safe)
1822 		__meltdown_safe = false;
1823 
1824 	/*
1825 	 * For reasons that aren't entirely clear, enabling KPTI on Cavium
1826 	 * ThunderX leads to apparent I-cache corruption of kernel text, which
1827 	 * ends as well as you might imagine. Don't even try. We cannot rely
1828 	 * on the cpus_have_*cap() helpers here to detect the CPU erratum
1829 	 * because cpucap detection order may change. However, since we know
1830 	 * affected CPUs are always in a homogeneous configuration, it is
1831 	 * safe to rely on this_cpu_has_cap() here.
1832 	 */
1833 	if (this_cpu_has_cap(ARM64_WORKAROUND_CAVIUM_27456)) {
1834 		str = "ARM64_WORKAROUND_CAVIUM_27456";
1835 		__kpti_forced = -1;
1836 	}
1837 
1838 	/* Useful for KASLR robustness */
1839 	if (kaslr_enabled() && kaslr_requires_kpti()) {
1840 		if (!__kpti_forced) {
1841 			str = "KASLR";
1842 			__kpti_forced = 1;
1843 		}
1844 	}
1845 
1846 	if (cpu_mitigations_off() && !__kpti_forced) {
1847 		str = "mitigations=off";
1848 		__kpti_forced = -1;
1849 	}
1850 
1851 	if (!IS_ENABLED(CONFIG_UNMAP_KERNEL_AT_EL0)) {
1852 		pr_info_once("kernel page table isolation disabled by kernel configuration\n");
1853 		return false;
1854 	}
1855 
1856 	/* Forced? */
1857 	if (__kpti_forced) {
1858 		pr_info_once("kernel page table isolation forced %s by %s\n",
1859 			     __kpti_forced > 0 ? "ON" : "OFF", str);
1860 		return __kpti_forced > 0;
1861 	}
1862 
1863 	return !meltdown_safe;
1864 }
1865 
has_nv1(const struct arm64_cpu_capabilities * entry,int scope)1866 static bool has_nv1(const struct arm64_cpu_capabilities *entry, int scope)
1867 {
1868 	/*
1869 	 * Although the Apple M2 family appears to support NV1, the
1870 	 * PTW barfs on the nVHE EL2 S1 page table format. Pretend
1871 	 * that it doesn't support NV1 at all.
1872 	 */
1873 	static const struct midr_range nv1_ni_list[] = {
1874 		MIDR_ALL_VERSIONS(MIDR_APPLE_M2_BLIZZARD),
1875 		MIDR_ALL_VERSIONS(MIDR_APPLE_M2_AVALANCHE),
1876 		MIDR_ALL_VERSIONS(MIDR_APPLE_M2_BLIZZARD_PRO),
1877 		MIDR_ALL_VERSIONS(MIDR_APPLE_M2_AVALANCHE_PRO),
1878 		MIDR_ALL_VERSIONS(MIDR_APPLE_M2_BLIZZARD_MAX),
1879 		MIDR_ALL_VERSIONS(MIDR_APPLE_M2_AVALANCHE_MAX),
1880 		{}
1881 	};
1882 
1883 	return (__system_matches_cap(ARM64_HAS_NESTED_VIRT) &&
1884 		!(has_cpuid_feature(entry, scope) ||
1885 		  is_midr_in_range_list(nv1_ni_list)));
1886 }
1887 
1888 #if defined(ID_AA64MMFR0_EL1_TGRAN_LPA2) && defined(ID_AA64MMFR0_EL1_TGRAN_2_SUPPORTED_LPA2)
has_lpa2_at_stage1(u64 mmfr0)1889 static bool has_lpa2_at_stage1(u64 mmfr0)
1890 {
1891 	unsigned int tgran;
1892 
1893 	tgran = cpuid_feature_extract_unsigned_field(mmfr0,
1894 					ID_AA64MMFR0_EL1_TGRAN_SHIFT);
1895 	return tgran == ID_AA64MMFR0_EL1_TGRAN_LPA2;
1896 }
1897 
has_lpa2_at_stage2(u64 mmfr0)1898 static bool has_lpa2_at_stage2(u64 mmfr0)
1899 {
1900 	unsigned int tgran;
1901 
1902 	tgran = cpuid_feature_extract_unsigned_field(mmfr0,
1903 					ID_AA64MMFR0_EL1_TGRAN_2_SHIFT);
1904 	return tgran == ID_AA64MMFR0_EL1_TGRAN_2_SUPPORTED_LPA2;
1905 }
1906 
has_lpa2(const struct arm64_cpu_capabilities * entry,int scope)1907 static bool has_lpa2(const struct arm64_cpu_capabilities *entry, int scope)
1908 {
1909 	u64 mmfr0;
1910 
1911 	mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
1912 	return has_lpa2_at_stage1(mmfr0) && has_lpa2_at_stage2(mmfr0);
1913 }
1914 #else
has_lpa2(const struct arm64_cpu_capabilities * entry,int scope)1915 static bool has_lpa2(const struct arm64_cpu_capabilities *entry, int scope)
1916 {
1917 	return false;
1918 }
1919 #endif
1920 
1921 #ifdef CONFIG_HW_PERF_EVENTS
has_pmuv3(const struct arm64_cpu_capabilities * entry,int scope)1922 static bool has_pmuv3(const struct arm64_cpu_capabilities *entry, int scope)
1923 {
1924 	u64 dfr0 = read_sanitised_ftr_reg(SYS_ID_AA64DFR0_EL1);
1925 	unsigned int pmuver;
1926 
1927 	/*
1928 	 * PMUVer follows the standard ID scheme for an unsigned field with the
1929 	 * exception of 0xF (IMP_DEF) which is treated specially and implies
1930 	 * FEAT_PMUv3 is not implemented.
1931 	 *
1932 	 * See DDI0487L.a D24.1.3.2 for more details.
1933 	 */
1934 	pmuver = cpuid_feature_extract_unsigned_field(dfr0,
1935 						      ID_AA64DFR0_EL1_PMUVer_SHIFT);
1936 	if (pmuver == ID_AA64DFR0_EL1_PMUVer_IMP_DEF)
1937 		return false;
1938 
1939 	return pmuver >= ID_AA64DFR0_EL1_PMUVer_IMP;
1940 }
1941 #endif
1942 
1943 #ifdef CONFIG_UNMAP_KERNEL_AT_EL0
1944 #define KPTI_NG_TEMP_VA		(-(1UL << PMD_SHIFT))
1945 
1946 extern
1947 void create_kpti_ng_temp_pgd(pgd_t *pgdir, phys_addr_t phys, unsigned long virt,
1948 			     phys_addr_t size, pgprot_t prot,
1949 			     phys_addr_t (*pgtable_alloc)(enum pgtable_type), int flags);
1950 
1951 static phys_addr_t __initdata kpti_ng_temp_alloc;
1952 
kpti_ng_pgd_alloc(enum pgtable_type type)1953 static phys_addr_t __init kpti_ng_pgd_alloc(enum pgtable_type type)
1954 {
1955 	kpti_ng_temp_alloc -= PAGE_SIZE;
1956 	return kpti_ng_temp_alloc;
1957 }
1958 
__kpti_install_ng_mappings(void * __unused)1959 static int __init __kpti_install_ng_mappings(void *__unused)
1960 {
1961 	typedef void (kpti_remap_fn)(int, int, phys_addr_t, unsigned long);
1962 	extern kpti_remap_fn idmap_kpti_install_ng_mappings;
1963 	kpti_remap_fn *remap_fn;
1964 
1965 	int cpu = smp_processor_id();
1966 	int levels = CONFIG_PGTABLE_LEVELS;
1967 	int order = order_base_2(levels);
1968 	u64 kpti_ng_temp_pgd_pa = 0;
1969 	pgd_t *kpti_ng_temp_pgd;
1970 	u64 alloc = 0;
1971 
1972 	if (levels == 5 && !pgtable_l5_enabled())
1973 		levels = 4;
1974 	else if (levels == 4 && !pgtable_l4_enabled())
1975 		levels = 3;
1976 
1977 	remap_fn = (void *)__pa_symbol(idmap_kpti_install_ng_mappings);
1978 
1979 	if (!cpu) {
1980 		alloc = __get_free_pages(GFP_ATOMIC | __GFP_ZERO, order);
1981 		kpti_ng_temp_pgd = (pgd_t *)(alloc + (levels - 1) * PAGE_SIZE);
1982 		kpti_ng_temp_alloc = kpti_ng_temp_pgd_pa = __pa(kpti_ng_temp_pgd);
1983 
1984 		//
1985 		// Create a minimal page table hierarchy that permits us to map
1986 		// the swapper page tables temporarily as we traverse them.
1987 		//
1988 		// The physical pages are laid out as follows:
1989 		//
1990 		// +--------+-/-------+-/------ +-/------ +-\\\--------+
1991 		// :  PTE[] : | PMD[] : | PUD[] : | P4D[] : ||| PGD[]  :
1992 		// +--------+-\-------+-\------ +-\------ +-///--------+
1993 		//      ^
1994 		// The first page is mapped into this hierarchy at a PMD_SHIFT
1995 		// aligned virtual address, so that we can manipulate the PTE
1996 		// level entries while the mapping is active. The first entry
1997 		// covers the PTE[] page itself, the remaining entries are free
1998 		// to be used as a ad-hoc fixmap.
1999 		//
2000 		create_kpti_ng_temp_pgd(kpti_ng_temp_pgd, __pa(alloc),
2001 					KPTI_NG_TEMP_VA, PAGE_SIZE, PAGE_KERNEL,
2002 					kpti_ng_pgd_alloc, 0);
2003 	}
2004 
2005 	cpu_install_idmap();
2006 	remap_fn(cpu, num_online_cpus(), kpti_ng_temp_pgd_pa, KPTI_NG_TEMP_VA);
2007 	cpu_uninstall_idmap();
2008 
2009 	if (!cpu) {
2010 		free_pages(alloc, order);
2011 		arm64_use_ng_mappings = true;
2012 	}
2013 
2014 	return 0;
2015 }
2016 
kpti_install_ng_mappings(void)2017 static void __init kpti_install_ng_mappings(void)
2018 {
2019 	/* Check whether KPTI is going to be used */
2020 	if (!arm64_kernel_unmapped_at_el0())
2021 		return;
2022 
2023 	/*
2024 	 * We don't need to rewrite the page-tables if either we've done
2025 	 * it already or we have KASLR enabled and therefore have not
2026 	 * created any global mappings at all.
2027 	 */
2028 	if (arm64_use_ng_mappings)
2029 		return;
2030 
2031 	stop_machine(__kpti_install_ng_mappings, NULL, cpu_online_mask);
2032 }
2033 
2034 #else
kpti_install_ng_mappings(void)2035 static inline void kpti_install_ng_mappings(void)
2036 {
2037 }
2038 #endif	/* CONFIG_UNMAP_KERNEL_AT_EL0 */
2039 
cpu_enable_kpti(struct arm64_cpu_capabilities const * cap)2040 static void cpu_enable_kpti(struct arm64_cpu_capabilities const *cap)
2041 {
2042 	if (__this_cpu_read(this_cpu_vector) == vectors) {
2043 		const char *v = arm64_get_bp_hardening_vector(EL1_VECTOR_KPTI);
2044 
2045 		__this_cpu_write(this_cpu_vector, v);
2046 	}
2047 
2048 }
2049 
parse_kpti(char * str)2050 static int __init parse_kpti(char *str)
2051 {
2052 	bool enabled;
2053 	int ret = kstrtobool(str, &enabled);
2054 
2055 	if (ret)
2056 		return ret;
2057 
2058 	__kpti_forced = enabled ? 1 : -1;
2059 	return 0;
2060 }
2061 early_param("kpti", parse_kpti);
2062 
2063 #ifdef CONFIG_ARM64_HW_AFDBM
2064 static struct cpumask dbm_cpus __read_mostly;
2065 
__cpu_enable_hw_dbm(void)2066 static inline void __cpu_enable_hw_dbm(void)
2067 {
2068 	u64 tcr = read_sysreg(tcr_el1) | TCR_HD;
2069 
2070 	write_sysreg(tcr, tcr_el1);
2071 	isb();
2072 	local_flush_tlb_all();
2073 }
2074 
cpu_has_broken_dbm(void)2075 static bool cpu_has_broken_dbm(void)
2076 {
2077 	/* List of CPUs which have broken DBM support. */
2078 	static const struct midr_range cpus[] = {
2079 #ifdef CONFIG_ARM64_ERRATUM_1024718
2080 		MIDR_ALL_VERSIONS(MIDR_CORTEX_A55),
2081 		/* Kryo4xx Silver (rdpe => r1p0) */
2082 		MIDR_REV(MIDR_QCOM_KRYO_4XX_SILVER, 0xd, 0xe),
2083 #endif
2084 #ifdef CONFIG_ARM64_ERRATUM_2051678
2085 		MIDR_REV_RANGE(MIDR_CORTEX_A510, 0, 0, 2),
2086 #endif
2087 		{},
2088 	};
2089 
2090 	return is_midr_in_range_list(cpus);
2091 }
2092 
cpu_can_use_dbm(const struct arm64_cpu_capabilities * cap)2093 static bool cpu_can_use_dbm(const struct arm64_cpu_capabilities *cap)
2094 {
2095 	return has_cpuid_feature(cap, SCOPE_LOCAL_CPU) &&
2096 	       !cpu_has_broken_dbm();
2097 }
2098 
cpu_enable_hw_dbm(struct arm64_cpu_capabilities const * cap)2099 static void cpu_enable_hw_dbm(struct arm64_cpu_capabilities const *cap)
2100 {
2101 	if (cpu_can_use_dbm(cap)) {
2102 		__cpu_enable_hw_dbm();
2103 		cpumask_set_cpu(smp_processor_id(), &dbm_cpus);
2104 	}
2105 }
2106 
has_hw_dbm(const struct arm64_cpu_capabilities * cap,int __unused)2107 static bool has_hw_dbm(const struct arm64_cpu_capabilities *cap,
2108 		       int __unused)
2109 {
2110 	/*
2111 	 * DBM is a non-conflicting feature. i.e, the kernel can safely
2112 	 * run a mix of CPUs with and without the feature. So, we
2113 	 * unconditionally enable the capability to allow any late CPU
2114 	 * to use the feature. We only enable the control bits on the
2115 	 * CPU, if it is supported.
2116 	 */
2117 
2118 	return true;
2119 }
2120 
2121 #endif
2122 
2123 #ifdef CONFIG_ARM64_AMU_EXTN
2124 
2125 /*
2126  * The "amu_cpus" cpumask only signals that the CPU implementation for the
2127  * flagged CPUs supports the Activity Monitors Unit (AMU) but does not provide
2128  * information regarding all the events that it supports. When a CPU bit is
2129  * set in the cpumask, the user of this feature can only rely on the presence
2130  * of the 4 fixed counters for that CPU. But this does not guarantee that the
2131  * counters are enabled or access to these counters is enabled by code
2132  * executed at higher exception levels (firmware).
2133  */
2134 static struct cpumask amu_cpus __read_mostly;
2135 
cpu_has_amu_feat(int cpu)2136 bool cpu_has_amu_feat(int cpu)
2137 {
2138 	return cpumask_test_cpu(cpu, &amu_cpus);
2139 }
2140 
get_cpu_with_amu_feat(void)2141 int get_cpu_with_amu_feat(void)
2142 {
2143 	return cpumask_any(&amu_cpus);
2144 }
2145 
cpu_amu_enable(struct arm64_cpu_capabilities const * cap)2146 static void cpu_amu_enable(struct arm64_cpu_capabilities const *cap)
2147 {
2148 	if (has_cpuid_feature(cap, SCOPE_LOCAL_CPU)) {
2149 		cpumask_set_cpu(smp_processor_id(), &amu_cpus);
2150 
2151 		/* 0 reference values signal broken/disabled counters */
2152 		if (!this_cpu_has_cap(ARM64_WORKAROUND_2457168))
2153 			update_freq_counters_refs();
2154 	}
2155 }
2156 
has_amu(const struct arm64_cpu_capabilities * cap,int __unused)2157 static bool has_amu(const struct arm64_cpu_capabilities *cap,
2158 		    int __unused)
2159 {
2160 	/*
2161 	 * The AMU extension is a non-conflicting feature: the kernel can
2162 	 * safely run a mix of CPUs with and without support for the
2163 	 * activity monitors extension. Therefore, unconditionally enable
2164 	 * the capability to allow any late CPU to use the feature.
2165 	 *
2166 	 * With this feature unconditionally enabled, the cpu_enable
2167 	 * function will be called for all CPUs that match the criteria,
2168 	 * including secondary and hotplugged, marking this feature as
2169 	 * present on that respective CPU. The enable function will also
2170 	 * print a detection message.
2171 	 */
2172 
2173 	return true;
2174 }
2175 #else
get_cpu_with_amu_feat(void)2176 int get_cpu_with_amu_feat(void)
2177 {
2178 	return nr_cpu_ids;
2179 }
2180 #endif
2181 
runs_at_el2(const struct arm64_cpu_capabilities * entry,int __unused)2182 static bool runs_at_el2(const struct arm64_cpu_capabilities *entry, int __unused)
2183 {
2184 	return is_kernel_in_hyp_mode();
2185 }
2186 
cpu_copy_el2regs(const struct arm64_cpu_capabilities * __unused)2187 static void cpu_copy_el2regs(const struct arm64_cpu_capabilities *__unused)
2188 {
2189 	/*
2190 	 * Copy register values that aren't redirected by hardware.
2191 	 *
2192 	 * Before code patching, we only set tpidr_el1, all CPUs need to copy
2193 	 * this value to tpidr_el2 before we patch the code. Once we've done
2194 	 * that, freshly-onlined CPUs will set tpidr_el2, so we don't need to
2195 	 * do anything here.
2196 	 */
2197 	if (!alternative_is_applied(ARM64_HAS_VIRT_HOST_EXTN))
2198 		write_sysreg(read_sysreg(tpidr_el1), tpidr_el2);
2199 }
2200 
has_nested_virt_support(const struct arm64_cpu_capabilities * cap,int scope)2201 static bool has_nested_virt_support(const struct arm64_cpu_capabilities *cap,
2202 				    int scope)
2203 {
2204 	if (kvm_get_mode() != KVM_MODE_NV)
2205 		return false;
2206 
2207 	if (!cpucap_multi_entry_cap_matches(cap, scope)) {
2208 		pr_warn("unavailable: %s\n", cap->desc);
2209 		return false;
2210 	}
2211 
2212 	return true;
2213 }
2214 
hvhe_possible(const struct arm64_cpu_capabilities * entry,int __unused)2215 static bool hvhe_possible(const struct arm64_cpu_capabilities *entry,
2216 			  int __unused)
2217 {
2218 	return arm64_test_sw_feature_override(ARM64_SW_FEATURE_OVERRIDE_HVHE);
2219 }
2220 
has_bbml2_noabort(const struct arm64_cpu_capabilities * caps,int scope)2221 static bool has_bbml2_noabort(const struct arm64_cpu_capabilities *caps, int scope)
2222 {
2223 	/*
2224 	 * We want to allow usage of BBML2 in as wide a range of kernel contexts
2225 	 * as possible. This list is therefore an allow-list of known-good
2226 	 * implementations that both support BBML2 and additionally, fulfill the
2227 	 * extra constraint of never generating TLB conflict aborts when using
2228 	 * the relaxed BBML2 semantics (such aborts make use of BBML2 in certain
2229 	 * kernel contexts difficult to prove safe against recursive aborts).
2230 	 *
2231 	 * Note that implementations can only be considered "known-good" if their
2232 	 * implementors attest to the fact that the implementation never raises
2233 	 * TLB conflict aborts for BBML2 mapping granularity changes.
2234 	 */
2235 	static const struct midr_range supports_bbml2_noabort_list[] = {
2236 		MIDR_REV_RANGE(MIDR_CORTEX_X4, 0, 3, 0xf),
2237 		MIDR_REV_RANGE(MIDR_NEOVERSE_V3, 0, 2, 0xf),
2238 		{}
2239 	};
2240 
2241 	/* Does our cpu guarantee to never raise TLB conflict aborts? */
2242 	if (!is_midr_in_range_list(supports_bbml2_noabort_list))
2243 		return false;
2244 
2245 	/*
2246 	 * We currently ignore the ID_AA64MMFR2_EL1 register, and only care
2247 	 * about whether the MIDR check passes.
2248 	 */
2249 
2250 	return true;
2251 }
2252 
2253 #ifdef CONFIG_ARM64_PAN
cpu_enable_pan(const struct arm64_cpu_capabilities * __unused)2254 static void cpu_enable_pan(const struct arm64_cpu_capabilities *__unused)
2255 {
2256 	/*
2257 	 * We modify PSTATE. This won't work from irq context as the PSTATE
2258 	 * is discarded once we return from the exception.
2259 	 */
2260 	WARN_ON_ONCE(in_interrupt());
2261 
2262 	sysreg_clear_set(sctlr_el1, SCTLR_EL1_SPAN, 0);
2263 	set_pstate_pan(1);
2264 }
2265 #endif /* CONFIG_ARM64_PAN */
2266 
2267 #ifdef CONFIG_ARM64_RAS_EXTN
cpu_clear_disr(const struct arm64_cpu_capabilities * __unused)2268 static void cpu_clear_disr(const struct arm64_cpu_capabilities *__unused)
2269 {
2270 	/* Firmware may have left a deferred SError in this register. */
2271 	write_sysreg_s(0, SYS_DISR_EL1);
2272 }
has_rasv1p1(const struct arm64_cpu_capabilities * __unused,int scope)2273 static bool has_rasv1p1(const struct arm64_cpu_capabilities *__unused, int scope)
2274 {
2275 	const struct arm64_cpu_capabilities rasv1p1_caps[] = {
2276 		{
2277 			ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, RAS, V1P1)
2278 		},
2279 		{
2280 			ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, RAS, IMP)
2281 		},
2282 		{
2283 			ARM64_CPUID_FIELDS(ID_AA64PFR1_EL1, RAS_frac, RASv1p1)
2284 		},
2285 	};
2286 
2287 	return (has_cpuid_feature(&rasv1p1_caps[0], scope) ||
2288 		(has_cpuid_feature(&rasv1p1_caps[1], scope) &&
2289 		 has_cpuid_feature(&rasv1p1_caps[2], scope)));
2290 }
2291 #endif /* CONFIG_ARM64_RAS_EXTN */
2292 
2293 #ifdef CONFIG_ARM64_PTR_AUTH
has_address_auth_cpucap(const struct arm64_cpu_capabilities * entry,int scope)2294 static bool has_address_auth_cpucap(const struct arm64_cpu_capabilities *entry, int scope)
2295 {
2296 	int boot_val, sec_val;
2297 
2298 	/* We don't expect to be called with SCOPE_SYSTEM */
2299 	WARN_ON(scope == SCOPE_SYSTEM);
2300 	/*
2301 	 * The ptr-auth feature levels are not intercompatible with lower
2302 	 * levels. Hence we must match ptr-auth feature level of the secondary
2303 	 * CPUs with that of the boot CPU. The level of boot cpu is fetched
2304 	 * from the sanitised register whereas direct register read is done for
2305 	 * the secondary CPUs.
2306 	 * The sanitised feature state is guaranteed to match that of the
2307 	 * boot CPU as a mismatched secondary CPU is parked before it gets
2308 	 * a chance to update the state, with the capability.
2309 	 */
2310 	boot_val = cpuid_feature_extract_field(read_sanitised_ftr_reg(entry->sys_reg),
2311 					       entry->field_pos, entry->sign);
2312 	if (scope & SCOPE_BOOT_CPU)
2313 		return boot_val >= entry->min_field_value;
2314 	/* Now check for the secondary CPUs with SCOPE_LOCAL_CPU scope */
2315 	sec_val = cpuid_feature_extract_field(__read_sysreg_by_encoding(entry->sys_reg),
2316 					      entry->field_pos, entry->sign);
2317 	return (sec_val >= entry->min_field_value) && (sec_val == boot_val);
2318 }
2319 
has_address_auth_metacap(const struct arm64_cpu_capabilities * entry,int scope)2320 static bool has_address_auth_metacap(const struct arm64_cpu_capabilities *entry,
2321 				     int scope)
2322 {
2323 	bool api = has_address_auth_cpucap(cpucap_ptrs[ARM64_HAS_ADDRESS_AUTH_IMP_DEF], scope);
2324 	bool apa = has_address_auth_cpucap(cpucap_ptrs[ARM64_HAS_ADDRESS_AUTH_ARCH_QARMA5], scope);
2325 	bool apa3 = has_address_auth_cpucap(cpucap_ptrs[ARM64_HAS_ADDRESS_AUTH_ARCH_QARMA3], scope);
2326 
2327 	return apa || apa3 || api;
2328 }
2329 
has_generic_auth(const struct arm64_cpu_capabilities * entry,int __unused)2330 static bool has_generic_auth(const struct arm64_cpu_capabilities *entry,
2331 			     int __unused)
2332 {
2333 	bool gpi = __system_matches_cap(ARM64_HAS_GENERIC_AUTH_IMP_DEF);
2334 	bool gpa = __system_matches_cap(ARM64_HAS_GENERIC_AUTH_ARCH_QARMA5);
2335 	bool gpa3 = __system_matches_cap(ARM64_HAS_GENERIC_AUTH_ARCH_QARMA3);
2336 
2337 	return gpa || gpa3 || gpi;
2338 }
2339 #endif /* CONFIG_ARM64_PTR_AUTH */
2340 
2341 #ifdef CONFIG_ARM64_E0PD
cpu_enable_e0pd(struct arm64_cpu_capabilities const * cap)2342 static void cpu_enable_e0pd(struct arm64_cpu_capabilities const *cap)
2343 {
2344 	if (this_cpu_has_cap(ARM64_HAS_E0PD))
2345 		sysreg_clear_set(tcr_el1, 0, TCR_E0PD1);
2346 }
2347 #endif /* CONFIG_ARM64_E0PD */
2348 
2349 #ifdef CONFIG_ARM64_PSEUDO_NMI
can_use_gic_priorities(const struct arm64_cpu_capabilities * entry,int scope)2350 static bool can_use_gic_priorities(const struct arm64_cpu_capabilities *entry,
2351 				   int scope)
2352 {
2353 	/*
2354 	 * ARM64_HAS_GICV3_CPUIF has a lower index, and is a boot CPU
2355 	 * feature, so will be detected earlier.
2356 	 */
2357 	BUILD_BUG_ON(ARM64_HAS_GIC_PRIO_MASKING <= ARM64_HAS_GICV3_CPUIF);
2358 	if (!cpus_have_cap(ARM64_HAS_GICV3_CPUIF))
2359 		return false;
2360 
2361 	return enable_pseudo_nmi;
2362 }
2363 
has_gic_prio_relaxed_sync(const struct arm64_cpu_capabilities * entry,int scope)2364 static bool has_gic_prio_relaxed_sync(const struct arm64_cpu_capabilities *entry,
2365 				      int scope)
2366 {
2367 	/*
2368 	 * If we're not using priority masking then we won't be poking PMR_EL1,
2369 	 * and there's no need to relax synchronization of writes to it, and
2370 	 * ICC_CTLR_EL1 might not be accessible and we must avoid reads from
2371 	 * that.
2372 	 *
2373 	 * ARM64_HAS_GIC_PRIO_MASKING has a lower index, and is a boot CPU
2374 	 * feature, so will be detected earlier.
2375 	 */
2376 	BUILD_BUG_ON(ARM64_HAS_GIC_PRIO_RELAXED_SYNC <= ARM64_HAS_GIC_PRIO_MASKING);
2377 	if (!cpus_have_cap(ARM64_HAS_GIC_PRIO_MASKING))
2378 		return false;
2379 
2380 	/*
2381 	 * When Priority Mask Hint Enable (PMHE) == 0b0, PMR is not used as a
2382 	 * hint for interrupt distribution, a DSB is not necessary when
2383 	 * unmasking IRQs via PMR, and we can relax the barrier to a NOP.
2384 	 *
2385 	 * Linux itself doesn't use 1:N distribution, so has no need to
2386 	 * set PMHE. The only reason to have it set is if EL3 requires it
2387 	 * (and we can't change it).
2388 	 */
2389 	return (gic_read_ctlr() & ICC_CTLR_EL1_PMHE_MASK) == 0;
2390 }
2391 #endif
2392 
2393 #ifdef CONFIG_ARM64_BTI
bti_enable(const struct arm64_cpu_capabilities * __unused)2394 static void bti_enable(const struct arm64_cpu_capabilities *__unused)
2395 {
2396 	/*
2397 	 * Use of X16/X17 for tail-calls and trampolines that jump to
2398 	 * function entry points using BR is a requirement for
2399 	 * marking binaries with GNU_PROPERTY_AARCH64_FEATURE_1_BTI.
2400 	 * So, be strict and forbid other BRs using other registers to
2401 	 * jump onto a PACIxSP instruction:
2402 	 */
2403 	sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_BT0 | SCTLR_EL1_BT1);
2404 	isb();
2405 }
2406 #endif /* CONFIG_ARM64_BTI */
2407 
2408 #ifdef CONFIG_ARM64_MTE
cpu_enable_mte(struct arm64_cpu_capabilities const * cap)2409 static void cpu_enable_mte(struct arm64_cpu_capabilities const *cap)
2410 {
2411 	sysreg_clear_set(sctlr_el1, 0, SCTLR_ELx_ATA | SCTLR_EL1_ATA0);
2412 
2413 	mte_cpu_setup();
2414 
2415 	/*
2416 	 * Clear the tags in the zero page. This needs to be done via the
2417 	 * linear map which has the Tagged attribute.
2418 	 */
2419 	if (try_page_mte_tagging(ZERO_PAGE(0))) {
2420 		mte_clear_page_tags(lm_alias(empty_zero_page));
2421 		set_page_mte_tagged(ZERO_PAGE(0));
2422 	}
2423 
2424 	kasan_init_hw_tags_cpu();
2425 }
2426 #endif /* CONFIG_ARM64_MTE */
2427 
user_feature_fixup(void)2428 static void user_feature_fixup(void)
2429 {
2430 	if (cpus_have_cap(ARM64_WORKAROUND_2658417)) {
2431 		struct arm64_ftr_reg *regp;
2432 
2433 		regp = get_arm64_ftr_reg(SYS_ID_AA64ISAR1_EL1);
2434 		if (regp)
2435 			regp->user_mask &= ~ID_AA64ISAR1_EL1_BF16_MASK;
2436 	}
2437 
2438 	if (cpus_have_cap(ARM64_WORKAROUND_SPECULATIVE_SSBS)) {
2439 		struct arm64_ftr_reg *regp;
2440 
2441 		regp = get_arm64_ftr_reg(SYS_ID_AA64PFR1_EL1);
2442 		if (regp)
2443 			regp->user_mask &= ~ID_AA64PFR1_EL1_SSBS_MASK;
2444 	}
2445 }
2446 
elf_hwcap_fixup(void)2447 static void elf_hwcap_fixup(void)
2448 {
2449 #ifdef CONFIG_COMPAT
2450 	if (cpus_have_cap(ARM64_WORKAROUND_1742098))
2451 		compat_elf_hwcap2 &= ~COMPAT_HWCAP2_AES;
2452 #endif /* CONFIG_COMPAT */
2453 }
2454 
2455 #ifdef CONFIG_KVM
is_kvm_protected_mode(const struct arm64_cpu_capabilities * entry,int __unused)2456 static bool is_kvm_protected_mode(const struct arm64_cpu_capabilities *entry, int __unused)
2457 {
2458 	return kvm_get_mode() == KVM_MODE_PROTECTED;
2459 }
2460 #endif /* CONFIG_KVM */
2461 
cpu_trap_el0_impdef(const struct arm64_cpu_capabilities * __unused)2462 static void cpu_trap_el0_impdef(const struct arm64_cpu_capabilities *__unused)
2463 {
2464 	sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_TIDCP);
2465 }
2466 
cpu_enable_dit(const struct arm64_cpu_capabilities * __unused)2467 static void cpu_enable_dit(const struct arm64_cpu_capabilities *__unused)
2468 {
2469 	set_pstate_dit(1);
2470 }
2471 
cpu_enable_mops(const struct arm64_cpu_capabilities * __unused)2472 static void cpu_enable_mops(const struct arm64_cpu_capabilities *__unused)
2473 {
2474 	sysreg_clear_set(sctlr_el1, 0, SCTLR_EL1_MSCEn);
2475 }
2476 
2477 #ifdef CONFIG_ARM64_POE
cpu_enable_poe(const struct arm64_cpu_capabilities * __unused)2478 static void cpu_enable_poe(const struct arm64_cpu_capabilities *__unused)
2479 {
2480 	sysreg_clear_set(REG_TCR2_EL1, 0, TCR2_EL1_E0POE);
2481 	sysreg_clear_set(CPACR_EL1, 0, CPACR_EL1_E0POE);
2482 }
2483 #endif
2484 
2485 #ifdef CONFIG_ARM64_GCS
cpu_enable_gcs(const struct arm64_cpu_capabilities * __unused)2486 static void cpu_enable_gcs(const struct arm64_cpu_capabilities *__unused)
2487 {
2488 	/* GCSPR_EL0 is always readable */
2489 	write_sysreg_s(GCSCRE0_EL1_nTR, SYS_GCSCRE0_EL1);
2490 }
2491 #endif
2492 
2493 /* Internal helper functions to match cpu capability type */
2494 static bool
cpucap_late_cpu_optional(const struct arm64_cpu_capabilities * cap)2495 cpucap_late_cpu_optional(const struct arm64_cpu_capabilities *cap)
2496 {
2497 	return !!(cap->type & ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU);
2498 }
2499 
2500 static bool
cpucap_late_cpu_permitted(const struct arm64_cpu_capabilities * cap)2501 cpucap_late_cpu_permitted(const struct arm64_cpu_capabilities *cap)
2502 {
2503 	return !!(cap->type & ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU);
2504 }
2505 
2506 static bool
cpucap_panic_on_conflict(const struct arm64_cpu_capabilities * cap)2507 cpucap_panic_on_conflict(const struct arm64_cpu_capabilities *cap)
2508 {
2509 	return !!(cap->type & ARM64_CPUCAP_PANIC_ON_CONFLICT);
2510 }
2511 
2512 static bool
test_has_mpam(const struct arm64_cpu_capabilities * entry,int scope)2513 test_has_mpam(const struct arm64_cpu_capabilities *entry, int scope)
2514 {
2515 	if (!has_cpuid_feature(entry, scope))
2516 		return false;
2517 
2518 	/* Check firmware actually enabled MPAM on this cpu. */
2519 	return (read_sysreg_s(SYS_MPAM1_EL1) & MPAM1_EL1_MPAMEN);
2520 }
2521 
2522 static void
cpu_enable_mpam(const struct arm64_cpu_capabilities * entry)2523 cpu_enable_mpam(const struct arm64_cpu_capabilities *entry)
2524 {
2525 	/*
2526 	 * Access by the kernel (at EL1) should use the reserved PARTID
2527 	 * which is configured unrestricted. This avoids priority-inversion
2528 	 * where latency sensitive tasks have to wait for a task that has
2529 	 * been throttled to release the lock.
2530 	 */
2531 	write_sysreg_s(0, SYS_MPAM1_EL1);
2532 }
2533 
2534 static bool
test_has_mpam_hcr(const struct arm64_cpu_capabilities * entry,int scope)2535 test_has_mpam_hcr(const struct arm64_cpu_capabilities *entry, int scope)
2536 {
2537 	u64 idr = read_sanitised_ftr_reg(SYS_MPAMIDR_EL1);
2538 
2539 	return idr & MPAMIDR_EL1_HAS_HCR;
2540 }
2541 
2542 static const struct arm64_cpu_capabilities arm64_features[] = {
2543 	{
2544 		.capability = ARM64_ALWAYS_BOOT,
2545 		.type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
2546 		.matches = has_always,
2547 	},
2548 	{
2549 		.capability = ARM64_ALWAYS_SYSTEM,
2550 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2551 		.matches = has_always,
2552 	},
2553 	{
2554 		.desc = "GICv3 CPU interface",
2555 		.capability = ARM64_HAS_GICV3_CPUIF,
2556 		.type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
2557 		.matches = has_useable_gicv3_cpuif,
2558 		ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, GIC, IMP)
2559 	},
2560 	{
2561 		.desc = "Enhanced Counter Virtualization",
2562 		.capability = ARM64_HAS_ECV,
2563 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2564 		.matches = has_cpuid_feature,
2565 		ARM64_CPUID_FIELDS(ID_AA64MMFR0_EL1, ECV, IMP)
2566 	},
2567 	{
2568 		.desc = "Enhanced Counter Virtualization (CNTPOFF)",
2569 		.capability = ARM64_HAS_ECV_CNTPOFF,
2570 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2571 		.matches = has_cpuid_feature,
2572 		ARM64_CPUID_FIELDS(ID_AA64MMFR0_EL1, ECV, CNTPOFF)
2573 	},
2574 #ifdef CONFIG_ARM64_PAN
2575 	{
2576 		.desc = "Privileged Access Never",
2577 		.capability = ARM64_HAS_PAN,
2578 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2579 		.matches = has_cpuid_feature,
2580 		.cpu_enable = cpu_enable_pan,
2581 		ARM64_CPUID_FIELDS(ID_AA64MMFR1_EL1, PAN, IMP)
2582 	},
2583 #endif /* CONFIG_ARM64_PAN */
2584 #ifdef CONFIG_ARM64_EPAN
2585 	{
2586 		.desc = "Enhanced Privileged Access Never",
2587 		.capability = ARM64_HAS_EPAN,
2588 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2589 		.matches = has_cpuid_feature,
2590 		ARM64_CPUID_FIELDS(ID_AA64MMFR1_EL1, PAN, PAN3)
2591 	},
2592 #endif /* CONFIG_ARM64_EPAN */
2593 #ifdef CONFIG_ARM64_LSE_ATOMICS
2594 	{
2595 		.desc = "LSE atomic instructions",
2596 		.capability = ARM64_HAS_LSE_ATOMICS,
2597 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2598 		.matches = has_cpuid_feature,
2599 		ARM64_CPUID_FIELDS(ID_AA64ISAR0_EL1, ATOMIC, IMP)
2600 	},
2601 #endif /* CONFIG_ARM64_LSE_ATOMICS */
2602 	{
2603 		.desc = "Virtualization Host Extensions",
2604 		.capability = ARM64_HAS_VIRT_HOST_EXTN,
2605 		.type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
2606 		.matches = runs_at_el2,
2607 		.cpu_enable = cpu_copy_el2regs,
2608 	},
2609 	{
2610 		.desc = "Nested Virtualization Support",
2611 		.capability = ARM64_HAS_NESTED_VIRT,
2612 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2613 		.matches = has_nested_virt_support,
2614 		.match_list = (const struct arm64_cpu_capabilities []){
2615 			{
2616 				.matches = has_cpuid_feature,
2617 				ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, NV, NV2)
2618 			},
2619 			{
2620 				.matches = has_cpuid_feature,
2621 				ARM64_CPUID_FIELDS(ID_AA64MMFR4_EL1, NV_frac, NV2_ONLY)
2622 			},
2623 			{ /* Sentinel */ }
2624 		},
2625 	},
2626 	{
2627 		.capability = ARM64_HAS_32BIT_EL0_DO_NOT_USE,
2628 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2629 		.matches = has_32bit_el0,
2630 		ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, EL0, AARCH32)
2631 	},
2632 #ifdef CONFIG_KVM
2633 	{
2634 		.desc = "32-bit EL1 Support",
2635 		.capability = ARM64_HAS_32BIT_EL1,
2636 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2637 		.matches = has_cpuid_feature,
2638 		ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, EL1, AARCH32)
2639 	},
2640 	{
2641 		.desc = "Protected KVM",
2642 		.capability = ARM64_KVM_PROTECTED_MODE,
2643 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2644 		.matches = is_kvm_protected_mode,
2645 	},
2646 	{
2647 		.desc = "HCRX_EL2 register",
2648 		.capability = ARM64_HAS_HCX,
2649 		.type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
2650 		.matches = has_cpuid_feature,
2651 		ARM64_CPUID_FIELDS(ID_AA64MMFR1_EL1, HCX, IMP)
2652 	},
2653 #endif
2654 	{
2655 		.desc = "Kernel page table isolation (KPTI)",
2656 		.capability = ARM64_UNMAP_KERNEL_AT_EL0,
2657 		.type = ARM64_CPUCAP_BOOT_RESTRICTED_CPU_LOCAL_FEATURE,
2658 		.cpu_enable = cpu_enable_kpti,
2659 		.matches = unmap_kernel_at_el0,
2660 		/*
2661 		 * The ID feature fields below are used to indicate that
2662 		 * the CPU doesn't need KPTI. See unmap_kernel_at_el0 for
2663 		 * more details.
2664 		 */
2665 		ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, CSV3, IMP)
2666 	},
2667 	{
2668 		.capability = ARM64_HAS_FPSIMD,
2669 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2670 		.matches = has_cpuid_feature,
2671 		.cpu_enable = cpu_enable_fpsimd,
2672 		ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, FP, IMP)
2673 	},
2674 #ifdef CONFIG_ARM64_PMEM
2675 	{
2676 		.desc = "Data cache clean to Point of Persistence",
2677 		.capability = ARM64_HAS_DCPOP,
2678 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2679 		.matches = has_cpuid_feature,
2680 		ARM64_CPUID_FIELDS(ID_AA64ISAR1_EL1, DPB, IMP)
2681 	},
2682 	{
2683 		.desc = "Data cache clean to Point of Deep Persistence",
2684 		.capability = ARM64_HAS_DCPODP,
2685 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2686 		.matches = has_cpuid_feature,
2687 		ARM64_CPUID_FIELDS(ID_AA64ISAR1_EL1, DPB, DPB2)
2688 	},
2689 #endif
2690 #ifdef CONFIG_ARM64_SVE
2691 	{
2692 		.desc = "Scalable Vector Extension",
2693 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2694 		.capability = ARM64_SVE,
2695 		.cpu_enable = cpu_enable_sve,
2696 		.matches = has_cpuid_feature,
2697 		ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, SVE, IMP)
2698 	},
2699 #endif /* CONFIG_ARM64_SVE */
2700 #ifdef CONFIG_ARM64_RAS_EXTN
2701 	{
2702 		.desc = "RAS Extension Support",
2703 		.capability = ARM64_HAS_RAS_EXTN,
2704 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2705 		.matches = has_cpuid_feature,
2706 		.cpu_enable = cpu_clear_disr,
2707 		ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, RAS, IMP)
2708 	},
2709 	{
2710 		.desc = "RASv1p1 Extension Support",
2711 		.capability = ARM64_HAS_RASV1P1_EXTN,
2712 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2713 		.matches = has_rasv1p1,
2714 	},
2715 #endif /* CONFIG_ARM64_RAS_EXTN */
2716 #ifdef CONFIG_ARM64_AMU_EXTN
2717 	{
2718 		.desc = "Activity Monitors Unit (AMU)",
2719 		.capability = ARM64_HAS_AMU_EXTN,
2720 		.type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE,
2721 		.matches = has_amu,
2722 		.cpu_enable = cpu_amu_enable,
2723 		.cpus = &amu_cpus,
2724 		ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, AMU, IMP)
2725 	},
2726 #endif /* CONFIG_ARM64_AMU_EXTN */
2727 	{
2728 		.desc = "Data cache clean to the PoU not required for I/D coherence",
2729 		.capability = ARM64_HAS_CACHE_IDC,
2730 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2731 		.matches = has_cache_idc,
2732 		.cpu_enable = cpu_emulate_effective_ctr,
2733 	},
2734 	{
2735 		.desc = "Instruction cache invalidation not required for I/D coherence",
2736 		.capability = ARM64_HAS_CACHE_DIC,
2737 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2738 		.matches = has_cache_dic,
2739 	},
2740 	{
2741 		.desc = "Stage-2 Force Write-Back",
2742 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2743 		.capability = ARM64_HAS_STAGE2_FWB,
2744 		.matches = has_cpuid_feature,
2745 		ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, FWB, IMP)
2746 	},
2747 	{
2748 		.desc = "ARMv8.4 Translation Table Level",
2749 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2750 		.capability = ARM64_HAS_ARMv8_4_TTL,
2751 		.matches = has_cpuid_feature,
2752 		ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, TTL, IMP)
2753 	},
2754 	{
2755 		.desc = "TLB range maintenance instructions",
2756 		.capability = ARM64_HAS_TLB_RANGE,
2757 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2758 		.matches = has_cpuid_feature,
2759 		ARM64_CPUID_FIELDS(ID_AA64ISAR0_EL1, TLB, RANGE)
2760 	},
2761 #ifdef CONFIG_ARM64_HW_AFDBM
2762 	{
2763 		.desc = "Hardware dirty bit management",
2764 		.type = ARM64_CPUCAP_WEAK_LOCAL_CPU_FEATURE,
2765 		.capability = ARM64_HW_DBM,
2766 		.matches = has_hw_dbm,
2767 		.cpu_enable = cpu_enable_hw_dbm,
2768 		.cpus = &dbm_cpus,
2769 		ARM64_CPUID_FIELDS(ID_AA64MMFR1_EL1, HAFDBS, DBM)
2770 	},
2771 #endif
2772 #ifdef CONFIG_ARM64_HAFT
2773 	{
2774 		.desc = "Hardware managed Access Flag for Table Descriptors",
2775 		/*
2776 		 * Contrary to the page/block access flag, the table access flag
2777 		 * cannot be emulated in software (no access fault will occur).
2778 		 * Therefore this should be used only if it's supported system
2779 		 * wide.
2780 		 */
2781 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2782 		.capability = ARM64_HAFT,
2783 		.matches = has_cpuid_feature,
2784 		ARM64_CPUID_FIELDS(ID_AA64MMFR1_EL1, HAFDBS, HAFT)
2785 	},
2786 #endif
2787 	{
2788 		.desc = "CRC32 instructions",
2789 		.capability = ARM64_HAS_CRC32,
2790 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2791 		.matches = has_cpuid_feature,
2792 		ARM64_CPUID_FIELDS(ID_AA64ISAR0_EL1, CRC32, IMP)
2793 	},
2794 	{
2795 		.desc = "Speculative Store Bypassing Safe (SSBS)",
2796 		.capability = ARM64_SSBS,
2797 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2798 		.matches = has_cpuid_feature,
2799 		ARM64_CPUID_FIELDS(ID_AA64PFR1_EL1, SSBS, IMP)
2800 	},
2801 #ifdef CONFIG_ARM64_CNP
2802 	{
2803 		.desc = "Common not Private translations",
2804 		.capability = ARM64_HAS_CNP,
2805 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2806 		.matches = has_useable_cnp,
2807 		.cpu_enable = cpu_enable_cnp,
2808 		ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, CnP, IMP)
2809 	},
2810 #endif
2811 	{
2812 		.desc = "Speculation barrier (SB)",
2813 		.capability = ARM64_HAS_SB,
2814 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2815 		.matches = has_cpuid_feature,
2816 		ARM64_CPUID_FIELDS(ID_AA64ISAR1_EL1, SB, IMP)
2817 	},
2818 #ifdef CONFIG_ARM64_PTR_AUTH
2819 	{
2820 		.desc = "Address authentication (architected QARMA5 algorithm)",
2821 		.capability = ARM64_HAS_ADDRESS_AUTH_ARCH_QARMA5,
2822 		.type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
2823 		.matches = has_address_auth_cpucap,
2824 		ARM64_CPUID_FIELDS(ID_AA64ISAR1_EL1, APA, PAuth)
2825 	},
2826 	{
2827 		.desc = "Address authentication (architected QARMA3 algorithm)",
2828 		.capability = ARM64_HAS_ADDRESS_AUTH_ARCH_QARMA3,
2829 		.type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
2830 		.matches = has_address_auth_cpucap,
2831 		ARM64_CPUID_FIELDS(ID_AA64ISAR2_EL1, APA3, PAuth)
2832 	},
2833 	{
2834 		.desc = "Address authentication (IMP DEF algorithm)",
2835 		.capability = ARM64_HAS_ADDRESS_AUTH_IMP_DEF,
2836 		.type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
2837 		.matches = has_address_auth_cpucap,
2838 		ARM64_CPUID_FIELDS(ID_AA64ISAR1_EL1, API, PAuth)
2839 	},
2840 	{
2841 		.capability = ARM64_HAS_ADDRESS_AUTH,
2842 		.type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
2843 		.matches = has_address_auth_metacap,
2844 	},
2845 	{
2846 		.desc = "Generic authentication (architected QARMA5 algorithm)",
2847 		.capability = ARM64_HAS_GENERIC_AUTH_ARCH_QARMA5,
2848 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2849 		.matches = has_cpuid_feature,
2850 		ARM64_CPUID_FIELDS(ID_AA64ISAR1_EL1, GPA, IMP)
2851 	},
2852 	{
2853 		.desc = "Generic authentication (architected QARMA3 algorithm)",
2854 		.capability = ARM64_HAS_GENERIC_AUTH_ARCH_QARMA3,
2855 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2856 		.matches = has_cpuid_feature,
2857 		ARM64_CPUID_FIELDS(ID_AA64ISAR2_EL1, GPA3, IMP)
2858 	},
2859 	{
2860 		.desc = "Generic authentication (IMP DEF algorithm)",
2861 		.capability = ARM64_HAS_GENERIC_AUTH_IMP_DEF,
2862 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2863 		.matches = has_cpuid_feature,
2864 		ARM64_CPUID_FIELDS(ID_AA64ISAR1_EL1, GPI, IMP)
2865 	},
2866 	{
2867 		.capability = ARM64_HAS_GENERIC_AUTH,
2868 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2869 		.matches = has_generic_auth,
2870 	},
2871 #endif /* CONFIG_ARM64_PTR_AUTH */
2872 #ifdef CONFIG_ARM64_PSEUDO_NMI
2873 	{
2874 		/*
2875 		 * Depends on having GICv3
2876 		 */
2877 		.desc = "IRQ priority masking",
2878 		.capability = ARM64_HAS_GIC_PRIO_MASKING,
2879 		.type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
2880 		.matches = can_use_gic_priorities,
2881 	},
2882 	{
2883 		/*
2884 		 * Depends on ARM64_HAS_GIC_PRIO_MASKING
2885 		 */
2886 		.capability = ARM64_HAS_GIC_PRIO_RELAXED_SYNC,
2887 		.type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
2888 		.matches = has_gic_prio_relaxed_sync,
2889 	},
2890 #endif
2891 #ifdef CONFIG_ARM64_E0PD
2892 	{
2893 		.desc = "E0PD",
2894 		.capability = ARM64_HAS_E0PD,
2895 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2896 		.cpu_enable = cpu_enable_e0pd,
2897 		.matches = has_cpuid_feature,
2898 		ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, E0PD, IMP)
2899 	},
2900 #endif
2901 	{
2902 		.desc = "Random Number Generator",
2903 		.capability = ARM64_HAS_RNG,
2904 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2905 		.matches = has_cpuid_feature,
2906 		ARM64_CPUID_FIELDS(ID_AA64ISAR0_EL1, RNDR, IMP)
2907 	},
2908 #ifdef CONFIG_ARM64_BTI
2909 	{
2910 		.desc = "Branch Target Identification",
2911 		.capability = ARM64_BTI,
2912 #ifdef CONFIG_ARM64_BTI_KERNEL
2913 		.type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
2914 #else
2915 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2916 #endif
2917 		.matches = has_cpuid_feature,
2918 		.cpu_enable = bti_enable,
2919 		ARM64_CPUID_FIELDS(ID_AA64PFR1_EL1, BT, IMP)
2920 	},
2921 #endif
2922 #ifdef CONFIG_ARM64_MTE
2923 	{
2924 		.desc = "Memory Tagging Extension",
2925 		.capability = ARM64_MTE,
2926 		.type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
2927 		.matches = has_cpuid_feature,
2928 		.cpu_enable = cpu_enable_mte,
2929 		ARM64_CPUID_FIELDS(ID_AA64PFR1_EL1, MTE, MTE2)
2930 	},
2931 	{
2932 		.desc = "Asymmetric MTE Tag Check Fault",
2933 		.capability = ARM64_MTE_ASYMM,
2934 		.type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
2935 		.matches = has_cpuid_feature,
2936 		ARM64_CPUID_FIELDS(ID_AA64PFR1_EL1, MTE, MTE3)
2937 	},
2938 	{
2939 		.desc = "FAR on MTE Tag Check Fault",
2940 		.capability = ARM64_MTE_FAR,
2941 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2942 		.matches = has_cpuid_feature,
2943 		ARM64_CPUID_FIELDS(ID_AA64PFR2_EL1, MTEFAR, IMP)
2944 	},
2945 	{
2946 		.desc = "Store Only MTE Tag Check",
2947 		.capability = ARM64_MTE_STORE_ONLY,
2948 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2949 		.matches = has_cpuid_feature,
2950 		ARM64_CPUID_FIELDS(ID_AA64PFR2_EL1, MTESTOREONLY, IMP)
2951 	},
2952 #endif /* CONFIG_ARM64_MTE */
2953 	{
2954 		.desc = "RCpc load-acquire (LDAPR)",
2955 		.capability = ARM64_HAS_LDAPR,
2956 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2957 		.matches = has_cpuid_feature,
2958 		ARM64_CPUID_FIELDS(ID_AA64ISAR1_EL1, LRCPC, IMP)
2959 	},
2960 	{
2961 		.desc = "Fine Grained Traps",
2962 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2963 		.capability = ARM64_HAS_FGT,
2964 		.matches = has_cpuid_feature,
2965 		ARM64_CPUID_FIELDS(ID_AA64MMFR0_EL1, FGT, IMP)
2966 	},
2967 	{
2968 		.desc = "Fine Grained Traps 2",
2969 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2970 		.capability = ARM64_HAS_FGT2,
2971 		.matches = has_cpuid_feature,
2972 		ARM64_CPUID_FIELDS(ID_AA64MMFR0_EL1, FGT, FGT2)
2973 	},
2974 #ifdef CONFIG_ARM64_SME
2975 	{
2976 		.desc = "Scalable Matrix Extension",
2977 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2978 		.capability = ARM64_SME,
2979 		.matches = has_cpuid_feature,
2980 		.cpu_enable = cpu_enable_sme,
2981 		ARM64_CPUID_FIELDS(ID_AA64PFR1_EL1, SME, IMP)
2982 	},
2983 	/* FA64 should be sorted after the base SME capability */
2984 	{
2985 		.desc = "FA64",
2986 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2987 		.capability = ARM64_SME_FA64,
2988 		.matches = has_cpuid_feature,
2989 		.cpu_enable = cpu_enable_fa64,
2990 		ARM64_CPUID_FIELDS(ID_AA64SMFR0_EL1, FA64, IMP)
2991 	},
2992 	{
2993 		.desc = "SME2",
2994 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
2995 		.capability = ARM64_SME2,
2996 		.matches = has_cpuid_feature,
2997 		.cpu_enable = cpu_enable_sme2,
2998 		ARM64_CPUID_FIELDS(ID_AA64PFR1_EL1, SME, SME2)
2999 	},
3000 #endif /* CONFIG_ARM64_SME */
3001 	{
3002 		.desc = "WFx with timeout",
3003 		.capability = ARM64_HAS_WFXT,
3004 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
3005 		.matches = has_cpuid_feature,
3006 		ARM64_CPUID_FIELDS(ID_AA64ISAR2_EL1, WFxT, IMP)
3007 	},
3008 	{
3009 		.desc = "Trap EL0 IMPLEMENTATION DEFINED functionality",
3010 		.capability = ARM64_HAS_TIDCP1,
3011 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
3012 		.matches = has_cpuid_feature,
3013 		.cpu_enable = cpu_trap_el0_impdef,
3014 		ARM64_CPUID_FIELDS(ID_AA64MMFR1_EL1, TIDCP1, IMP)
3015 	},
3016 	{
3017 		.desc = "Data independent timing control (DIT)",
3018 		.capability = ARM64_HAS_DIT,
3019 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
3020 		.matches = has_cpuid_feature,
3021 		.cpu_enable = cpu_enable_dit,
3022 		ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, DIT, IMP)
3023 	},
3024 	{
3025 		.desc = "Memory Copy and Memory Set instructions",
3026 		.capability = ARM64_HAS_MOPS,
3027 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
3028 		.matches = has_cpuid_feature,
3029 		.cpu_enable = cpu_enable_mops,
3030 		ARM64_CPUID_FIELDS(ID_AA64ISAR2_EL1, MOPS, IMP)
3031 	},
3032 	{
3033 		.capability = ARM64_HAS_TCR2,
3034 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
3035 		.matches = has_cpuid_feature,
3036 		ARM64_CPUID_FIELDS(ID_AA64MMFR3_EL1, TCRX, IMP)
3037 	},
3038 	{
3039 		.desc = "Stage-1 Permission Indirection Extension (S1PIE)",
3040 		.capability = ARM64_HAS_S1PIE,
3041 		.type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
3042 		.matches = has_cpuid_feature,
3043 		ARM64_CPUID_FIELDS(ID_AA64MMFR3_EL1, S1PIE, IMP)
3044 	},
3045 	{
3046 		.desc = "VHE for hypervisor only",
3047 		.capability = ARM64_KVM_HVHE,
3048 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
3049 		.matches = hvhe_possible,
3050 	},
3051 	{
3052 		.desc = "Enhanced Virtualization Traps",
3053 		.capability = ARM64_HAS_EVT,
3054 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
3055 		.matches = has_cpuid_feature,
3056 		ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, EVT, IMP)
3057 	},
3058 	{
3059 		.desc = "BBM Level 2 without TLB conflict abort",
3060 		.capability = ARM64_HAS_BBML2_NOABORT,
3061 		.type = ARM64_CPUCAP_EARLY_LOCAL_CPU_FEATURE,
3062 		.matches = has_bbml2_noabort,
3063 	},
3064 	{
3065 		.desc = "52-bit Virtual Addressing for KVM (LPA2)",
3066 		.capability = ARM64_HAS_LPA2,
3067 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
3068 		.matches = has_lpa2,
3069 	},
3070 	{
3071 		.desc = "FPMR",
3072 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
3073 		.capability = ARM64_HAS_FPMR,
3074 		.matches = has_cpuid_feature,
3075 		.cpu_enable = cpu_enable_fpmr,
3076 		ARM64_CPUID_FIELDS(ID_AA64PFR2_EL1, FPMR, IMP)
3077 	},
3078 #ifdef CONFIG_ARM64_VA_BITS_52
3079 	{
3080 		.capability = ARM64_HAS_VA52,
3081 		.type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
3082 		.matches = has_cpuid_feature,
3083 #ifdef CONFIG_ARM64_64K_PAGES
3084 		.desc = "52-bit Virtual Addressing (LVA)",
3085 		ARM64_CPUID_FIELDS(ID_AA64MMFR2_EL1, VARange, 52)
3086 #else
3087 		.desc = "52-bit Virtual Addressing (LPA2)",
3088 #ifdef CONFIG_ARM64_4K_PAGES
3089 		ARM64_CPUID_FIELDS(ID_AA64MMFR0_EL1, TGRAN4, 52_BIT)
3090 #else
3091 		ARM64_CPUID_FIELDS(ID_AA64MMFR0_EL1, TGRAN16, 52_BIT)
3092 #endif
3093 #endif
3094 	},
3095 #endif
3096 	{
3097 		.desc = "Memory Partitioning And Monitoring",
3098 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
3099 		.capability = ARM64_MPAM,
3100 		.matches = test_has_mpam,
3101 		.cpu_enable = cpu_enable_mpam,
3102 		ARM64_CPUID_FIELDS(ID_AA64PFR0_EL1, MPAM, 1)
3103 	},
3104 	{
3105 		.desc = "Memory Partitioning And Monitoring Virtualisation",
3106 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
3107 		.capability = ARM64_MPAM_HCR,
3108 		.matches = test_has_mpam_hcr,
3109 	},
3110 	{
3111 		.desc = "NV1",
3112 		.capability = ARM64_HAS_HCR_NV1,
3113 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
3114 		.matches = has_nv1,
3115 		ARM64_CPUID_FIELDS_NEG(ID_AA64MMFR4_EL1, E2H0, NI_NV1)
3116 	},
3117 #ifdef CONFIG_ARM64_POE
3118 	{
3119 		.desc = "Stage-1 Permission Overlay Extension (S1POE)",
3120 		.capability = ARM64_HAS_S1POE,
3121 		.type = ARM64_CPUCAP_BOOT_CPU_FEATURE,
3122 		.matches = has_cpuid_feature,
3123 		.cpu_enable = cpu_enable_poe,
3124 		ARM64_CPUID_FIELDS(ID_AA64MMFR3_EL1, S1POE, IMP)
3125 	},
3126 #endif
3127 #ifdef CONFIG_ARM64_GCS
3128 	{
3129 		.desc = "Guarded Control Stack (GCS)",
3130 		.capability = ARM64_HAS_GCS,
3131 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
3132 		.cpu_enable = cpu_enable_gcs,
3133 		.matches = has_cpuid_feature,
3134 		ARM64_CPUID_FIELDS(ID_AA64PFR1_EL1, GCS, IMP)
3135 	},
3136 #endif
3137 #ifdef CONFIG_HW_PERF_EVENTS
3138 	{
3139 		.desc = "PMUv3",
3140 		.capability = ARM64_HAS_PMUV3,
3141 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
3142 		.matches = has_pmuv3,
3143 	},
3144 #endif
3145 	{
3146 		.desc = "SCTLR2",
3147 		.capability = ARM64_HAS_SCTLR2,
3148 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,
3149 		.matches = has_cpuid_feature,
3150 		ARM64_CPUID_FIELDS(ID_AA64MMFR3_EL1, SCTLRX, IMP)
3151 	},
3152 	{
3153 		.desc = "GICv5 CPU interface",
3154 		.type = ARM64_CPUCAP_STRICT_BOOT_CPU_FEATURE,
3155 		.capability = ARM64_HAS_GICV5_CPUIF,
3156 		.matches = has_cpuid_feature,
3157 		ARM64_CPUID_FIELDS(ID_AA64PFR2_EL1, GCIE, IMP)
3158 	},
3159 	{},
3160 };
3161 
3162 #define HWCAP_CPUID_MATCH(reg, field, min_value)			\
3163 		.matches = has_user_cpuid_feature,			\
3164 		ARM64_CPUID_FIELDS(reg, field, min_value)
3165 
3166 #define __HWCAP_CAP(name, cap_type, cap)					\
3167 		.desc = name,							\
3168 		.type = ARM64_CPUCAP_SYSTEM_FEATURE,				\
3169 		.hwcap_type = cap_type,						\
3170 		.hwcap = cap,							\
3171 
3172 #define HWCAP_CAP(reg, field, min_value, cap_type, cap)		\
3173 	{									\
3174 		__HWCAP_CAP(#cap, cap_type, cap)				\
3175 		HWCAP_CPUID_MATCH(reg, field, min_value) 		\
3176 	}
3177 
3178 #define HWCAP_MULTI_CAP(list, cap_type, cap)					\
3179 	{									\
3180 		__HWCAP_CAP(#cap, cap_type, cap)				\
3181 		.matches = cpucap_multi_entry_cap_matches,			\
3182 		.match_list = list,						\
3183 	}
3184 
3185 #define HWCAP_CAP_MATCH(match, cap_type, cap)					\
3186 	{									\
3187 		__HWCAP_CAP(#cap, cap_type, cap)				\
3188 		.matches = match,						\
3189 	}
3190 
3191 #define HWCAP_CAP_MATCH_ID(match, reg, field, min_value, cap_type, cap)		\
3192 	{									\
3193 		__HWCAP_CAP(#cap, cap_type, cap)				\
3194 		HWCAP_CPUID_MATCH(reg, field, min_value) 			\
3195 		.matches = match,						\
3196 	}
3197 
3198 #ifdef CONFIG_ARM64_PTR_AUTH
3199 static const struct arm64_cpu_capabilities ptr_auth_hwcap_addr_matches[] = {
3200 	{
3201 		HWCAP_CPUID_MATCH(ID_AA64ISAR1_EL1, APA, PAuth)
3202 	},
3203 	{
3204 		HWCAP_CPUID_MATCH(ID_AA64ISAR2_EL1, APA3, PAuth)
3205 	},
3206 	{
3207 		HWCAP_CPUID_MATCH(ID_AA64ISAR1_EL1, API, PAuth)
3208 	},
3209 	{},
3210 };
3211 
3212 static const struct arm64_cpu_capabilities ptr_auth_hwcap_gen_matches[] = {
3213 	{
3214 		HWCAP_CPUID_MATCH(ID_AA64ISAR1_EL1, GPA, IMP)
3215 	},
3216 	{
3217 		HWCAP_CPUID_MATCH(ID_AA64ISAR2_EL1, GPA3, IMP)
3218 	},
3219 	{
3220 		HWCAP_CPUID_MATCH(ID_AA64ISAR1_EL1, GPI, IMP)
3221 	},
3222 	{},
3223 };
3224 #endif
3225 
3226 #ifdef CONFIG_ARM64_SVE
has_sve_feature(const struct arm64_cpu_capabilities * cap,int scope)3227 static bool has_sve_feature(const struct arm64_cpu_capabilities *cap, int scope)
3228 {
3229 	return system_supports_sve() && has_user_cpuid_feature(cap, scope);
3230 }
3231 #endif
3232 
3233 #ifdef CONFIG_ARM64_SME
has_sme_feature(const struct arm64_cpu_capabilities * cap,int scope)3234 static bool has_sme_feature(const struct arm64_cpu_capabilities *cap, int scope)
3235 {
3236 	return system_supports_sme() && has_user_cpuid_feature(cap, scope);
3237 }
3238 #endif
3239 
3240 static const struct arm64_cpu_capabilities arm64_elf_hwcaps[] = {
3241 	HWCAP_CAP(ID_AA64ISAR0_EL1, AES, PMULL, CAP_HWCAP, KERNEL_HWCAP_PMULL),
3242 	HWCAP_CAP(ID_AA64ISAR0_EL1, AES, AES, CAP_HWCAP, KERNEL_HWCAP_AES),
3243 	HWCAP_CAP(ID_AA64ISAR0_EL1, SHA1, IMP, CAP_HWCAP, KERNEL_HWCAP_SHA1),
3244 	HWCAP_CAP(ID_AA64ISAR0_EL1, SHA2, SHA256, CAP_HWCAP, KERNEL_HWCAP_SHA2),
3245 	HWCAP_CAP(ID_AA64ISAR0_EL1, SHA2, SHA512, CAP_HWCAP, KERNEL_HWCAP_SHA512),
3246 	HWCAP_CAP(ID_AA64ISAR0_EL1, CRC32, IMP, CAP_HWCAP, KERNEL_HWCAP_CRC32),
3247 	HWCAP_CAP(ID_AA64ISAR0_EL1, ATOMIC, IMP, CAP_HWCAP, KERNEL_HWCAP_ATOMICS),
3248 	HWCAP_CAP(ID_AA64ISAR0_EL1, ATOMIC, FEAT_LSE128, CAP_HWCAP, KERNEL_HWCAP_LSE128),
3249 	HWCAP_CAP(ID_AA64ISAR0_EL1, RDM, IMP, CAP_HWCAP, KERNEL_HWCAP_ASIMDRDM),
3250 	HWCAP_CAP(ID_AA64ISAR0_EL1, SHA3, IMP, CAP_HWCAP, KERNEL_HWCAP_SHA3),
3251 	HWCAP_CAP(ID_AA64ISAR0_EL1, SM3, IMP, CAP_HWCAP, KERNEL_HWCAP_SM3),
3252 	HWCAP_CAP(ID_AA64ISAR0_EL1, SM4, IMP, CAP_HWCAP, KERNEL_HWCAP_SM4),
3253 	HWCAP_CAP(ID_AA64ISAR0_EL1, DP, IMP, CAP_HWCAP, KERNEL_HWCAP_ASIMDDP),
3254 	HWCAP_CAP(ID_AA64ISAR0_EL1, FHM, IMP, CAP_HWCAP, KERNEL_HWCAP_ASIMDFHM),
3255 	HWCAP_CAP(ID_AA64ISAR0_EL1, TS, FLAGM, CAP_HWCAP, KERNEL_HWCAP_FLAGM),
3256 	HWCAP_CAP(ID_AA64ISAR0_EL1, TS, FLAGM2, CAP_HWCAP, KERNEL_HWCAP_FLAGM2),
3257 	HWCAP_CAP(ID_AA64ISAR0_EL1, RNDR, IMP, CAP_HWCAP, KERNEL_HWCAP_RNG),
3258 	HWCAP_CAP(ID_AA64ISAR3_EL1, FPRCVT, IMP, CAP_HWCAP, KERNEL_HWCAP_FPRCVT),
3259 	HWCAP_CAP(ID_AA64PFR0_EL1, FP, IMP, CAP_HWCAP, KERNEL_HWCAP_FP),
3260 	HWCAP_CAP(ID_AA64PFR0_EL1, FP, FP16, CAP_HWCAP, KERNEL_HWCAP_FPHP),
3261 	HWCAP_CAP(ID_AA64PFR0_EL1, AdvSIMD, IMP, CAP_HWCAP, KERNEL_HWCAP_ASIMD),
3262 	HWCAP_CAP(ID_AA64PFR0_EL1, AdvSIMD, FP16, CAP_HWCAP, KERNEL_HWCAP_ASIMDHP),
3263 	HWCAP_CAP(ID_AA64PFR0_EL1, DIT, IMP, CAP_HWCAP, KERNEL_HWCAP_DIT),
3264 	HWCAP_CAP(ID_AA64PFR2_EL1, FPMR, IMP, CAP_HWCAP, KERNEL_HWCAP_FPMR),
3265 	HWCAP_CAP(ID_AA64ISAR1_EL1, DPB, IMP, CAP_HWCAP, KERNEL_HWCAP_DCPOP),
3266 	HWCAP_CAP(ID_AA64ISAR1_EL1, DPB, DPB2, CAP_HWCAP, KERNEL_HWCAP_DCPODP),
3267 	HWCAP_CAP(ID_AA64ISAR1_EL1, JSCVT, IMP, CAP_HWCAP, KERNEL_HWCAP_JSCVT),
3268 	HWCAP_CAP(ID_AA64ISAR1_EL1, FCMA, IMP, CAP_HWCAP, KERNEL_HWCAP_FCMA),
3269 	HWCAP_CAP(ID_AA64ISAR1_EL1, LRCPC, IMP, CAP_HWCAP, KERNEL_HWCAP_LRCPC),
3270 	HWCAP_CAP(ID_AA64ISAR1_EL1, LRCPC, LRCPC2, CAP_HWCAP, KERNEL_HWCAP_ILRCPC),
3271 	HWCAP_CAP(ID_AA64ISAR1_EL1, LRCPC, LRCPC3, CAP_HWCAP, KERNEL_HWCAP_LRCPC3),
3272 	HWCAP_CAP(ID_AA64ISAR1_EL1, FRINTTS, IMP, CAP_HWCAP, KERNEL_HWCAP_FRINT),
3273 	HWCAP_CAP(ID_AA64ISAR1_EL1, SB, IMP, CAP_HWCAP, KERNEL_HWCAP_SB),
3274 	HWCAP_CAP(ID_AA64ISAR1_EL1, BF16, IMP, CAP_HWCAP, KERNEL_HWCAP_BF16),
3275 	HWCAP_CAP(ID_AA64ISAR1_EL1, BF16, EBF16, CAP_HWCAP, KERNEL_HWCAP_EBF16),
3276 	HWCAP_CAP(ID_AA64ISAR1_EL1, DGH, IMP, CAP_HWCAP, KERNEL_HWCAP_DGH),
3277 	HWCAP_CAP(ID_AA64ISAR1_EL1, I8MM, IMP, CAP_HWCAP, KERNEL_HWCAP_I8MM),
3278 	HWCAP_CAP(ID_AA64ISAR2_EL1, LUT, IMP, CAP_HWCAP, KERNEL_HWCAP_LUT),
3279 	HWCAP_CAP(ID_AA64ISAR3_EL1, FAMINMAX, IMP, CAP_HWCAP, KERNEL_HWCAP_FAMINMAX),
3280 	HWCAP_CAP(ID_AA64MMFR2_EL1, AT, IMP, CAP_HWCAP, KERNEL_HWCAP_USCAT),
3281 #ifdef CONFIG_ARM64_SVE
3282 	HWCAP_CAP(ID_AA64PFR0_EL1, SVE, IMP, CAP_HWCAP, KERNEL_HWCAP_SVE),
3283 	HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SVEver, SVE2p2, CAP_HWCAP, KERNEL_HWCAP_SVE2P2),
3284 	HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SVEver, SVE2p1, CAP_HWCAP, KERNEL_HWCAP_SVE2P1),
3285 	HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SVEver, SVE2, CAP_HWCAP, KERNEL_HWCAP_SVE2),
3286 	HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, AES, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEAES),
3287 	HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, AES, PMULL128, CAP_HWCAP, KERNEL_HWCAP_SVEPMULL),
3288 	HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, AES, AES2, CAP_HWCAP, KERNEL_HWCAP_SVE_AES2),
3289 	HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, BitPerm, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEBITPERM),
3290 	HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, B16B16, IMP, CAP_HWCAP, KERNEL_HWCAP_SVE_B16B16),
3291 	HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, B16B16, BFSCALE, CAP_HWCAP, KERNEL_HWCAP_SVE_BFSCALE),
3292 	HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, BF16, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEBF16),
3293 	HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, BF16, EBF16, CAP_HWCAP, KERNEL_HWCAP_SVE_EBF16),
3294 	HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SHA3, IMP, CAP_HWCAP, KERNEL_HWCAP_SVESHA3),
3295 	HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, SM4, IMP, CAP_HWCAP, KERNEL_HWCAP_SVESM4),
3296 	HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, I8MM, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEI8MM),
3297 	HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, F32MM, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEF32MM),
3298 	HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, F64MM, IMP, CAP_HWCAP, KERNEL_HWCAP_SVEF64MM),
3299 	HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, F16MM, IMP, CAP_HWCAP, KERNEL_HWCAP_SVE_F16MM),
3300 	HWCAP_CAP_MATCH_ID(has_sve_feature, ID_AA64ZFR0_EL1, EltPerm, IMP, CAP_HWCAP, KERNEL_HWCAP_SVE_ELTPERM),
3301 #endif
3302 #ifdef CONFIG_ARM64_GCS
3303 	HWCAP_CAP(ID_AA64PFR1_EL1, GCS, IMP, CAP_HWCAP, KERNEL_HWCAP_GCS),
3304 #endif
3305 	HWCAP_CAP(ID_AA64PFR1_EL1, SSBS, SSBS2, CAP_HWCAP, KERNEL_HWCAP_SSBS),
3306 #ifdef CONFIG_ARM64_BTI
3307 	HWCAP_CAP(ID_AA64PFR1_EL1, BT, IMP, CAP_HWCAP, KERNEL_HWCAP_BTI),
3308 #endif
3309 #ifdef CONFIG_ARM64_PTR_AUTH
3310 	HWCAP_MULTI_CAP(ptr_auth_hwcap_addr_matches, CAP_HWCAP, KERNEL_HWCAP_PACA),
3311 	HWCAP_MULTI_CAP(ptr_auth_hwcap_gen_matches, CAP_HWCAP, KERNEL_HWCAP_PACG),
3312 #endif
3313 #ifdef CONFIG_ARM64_MTE
3314 	HWCAP_CAP(ID_AA64PFR1_EL1, MTE, MTE2, CAP_HWCAP, KERNEL_HWCAP_MTE),
3315 	HWCAP_CAP(ID_AA64PFR1_EL1, MTE, MTE3, CAP_HWCAP, KERNEL_HWCAP_MTE3),
3316 	HWCAP_CAP(ID_AA64PFR2_EL1, MTEFAR, IMP, CAP_HWCAP, KERNEL_HWCAP_MTE_FAR),
3317 	HWCAP_CAP(ID_AA64PFR2_EL1, MTESTOREONLY, IMP, CAP_HWCAP , KERNEL_HWCAP_MTE_STORE_ONLY),
3318 #endif /* CONFIG_ARM64_MTE */
3319 	HWCAP_CAP(ID_AA64MMFR0_EL1, ECV, IMP, CAP_HWCAP, KERNEL_HWCAP_ECV),
3320 	HWCAP_CAP(ID_AA64MMFR1_EL1, AFP, IMP, CAP_HWCAP, KERNEL_HWCAP_AFP),
3321 	HWCAP_CAP(ID_AA64ISAR2_EL1, CSSC, IMP, CAP_HWCAP, KERNEL_HWCAP_CSSC),
3322 	HWCAP_CAP(ID_AA64ISAR2_EL1, CSSC, CMPBR, CAP_HWCAP, KERNEL_HWCAP_CMPBR),
3323 	HWCAP_CAP(ID_AA64ISAR2_EL1, RPRFM, IMP, CAP_HWCAP, KERNEL_HWCAP_RPRFM),
3324 	HWCAP_CAP(ID_AA64ISAR2_EL1, RPRES, IMP, CAP_HWCAP, KERNEL_HWCAP_RPRES),
3325 	HWCAP_CAP(ID_AA64ISAR2_EL1, WFxT, IMP, CAP_HWCAP, KERNEL_HWCAP_WFXT),
3326 	HWCAP_CAP(ID_AA64ISAR2_EL1, MOPS, IMP, CAP_HWCAP, KERNEL_HWCAP_MOPS),
3327 	HWCAP_CAP(ID_AA64ISAR2_EL1, BC, IMP, CAP_HWCAP, KERNEL_HWCAP_HBC),
3328 #ifdef CONFIG_ARM64_SME
3329 	HWCAP_CAP(ID_AA64PFR1_EL1, SME, IMP, CAP_HWCAP, KERNEL_HWCAP_SME),
3330 	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, FA64, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_FA64),
3331 	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, LUTv2, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_LUTV2),
3332 	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2p2, CAP_HWCAP, KERNEL_HWCAP_SME2P2),
3333 	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2p1, CAP_HWCAP, KERNEL_HWCAP_SME2P1),
3334 	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMEver, SME2, CAP_HWCAP, KERNEL_HWCAP_SME2),
3335 	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, I16I64, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_I16I64),
3336 	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, F64F64, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F64F64),
3337 	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, I16I32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_I16I32),
3338 	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, B16B16, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_B16B16),
3339 	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, F16F16, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F16F16),
3340 	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, F8F16, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F8F16),
3341 	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, F8F32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F8F32),
3342 	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, I8I32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_I8I32),
3343 	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, F16F32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F16F32),
3344 	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, B16F32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_B16F32),
3345 	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, BI32I32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_BI32I32),
3346 	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, F32F32, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_F32F32),
3347 	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SF8FMA, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_SF8FMA),
3348 	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SF8DP4, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_SF8DP4),
3349 	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SF8DP2, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_SF8DP2),
3350 	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SBitPerm, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_SBITPERM),
3351 	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, AES, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_AES),
3352 	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SFEXPA, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_SFEXPA),
3353 	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, STMOP, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_STMOP),
3354 	HWCAP_CAP_MATCH_ID(has_sme_feature, ID_AA64SMFR0_EL1, SMOP4, IMP, CAP_HWCAP, KERNEL_HWCAP_SME_SMOP4),
3355 #endif /* CONFIG_ARM64_SME */
3356 	HWCAP_CAP(ID_AA64FPFR0_EL1, F8CVT, IMP, CAP_HWCAP, KERNEL_HWCAP_F8CVT),
3357 	HWCAP_CAP(ID_AA64FPFR0_EL1, F8FMA, IMP, CAP_HWCAP, KERNEL_HWCAP_F8FMA),
3358 	HWCAP_CAP(ID_AA64FPFR0_EL1, F8DP4, IMP, CAP_HWCAP, KERNEL_HWCAP_F8DP4),
3359 	HWCAP_CAP(ID_AA64FPFR0_EL1, F8DP2, IMP, CAP_HWCAP, KERNEL_HWCAP_F8DP2),
3360 	HWCAP_CAP(ID_AA64FPFR0_EL1, F8MM8, IMP, CAP_HWCAP, KERNEL_HWCAP_F8MM8),
3361 	HWCAP_CAP(ID_AA64FPFR0_EL1, F8MM4, IMP, CAP_HWCAP, KERNEL_HWCAP_F8MM4),
3362 	HWCAP_CAP(ID_AA64FPFR0_EL1, F8E4M3, IMP, CAP_HWCAP, KERNEL_HWCAP_F8E4M3),
3363 	HWCAP_CAP(ID_AA64FPFR0_EL1, F8E5M2, IMP, CAP_HWCAP, KERNEL_HWCAP_F8E5M2),
3364 #ifdef CONFIG_ARM64_POE
3365 	HWCAP_CAP(ID_AA64MMFR3_EL1, S1POE, IMP, CAP_HWCAP, KERNEL_HWCAP_POE),
3366 #endif
3367 	{},
3368 };
3369 
3370 #ifdef CONFIG_COMPAT
compat_has_neon(const struct arm64_cpu_capabilities * cap,int scope)3371 static bool compat_has_neon(const struct arm64_cpu_capabilities *cap, int scope)
3372 {
3373 	/*
3374 	 * Check that all of MVFR1_EL1.{SIMDSP, SIMDInt, SIMDLS} are available,
3375 	 * in line with that of arm32 as in vfp_init(). We make sure that the
3376 	 * check is future proof, by making sure value is non-zero.
3377 	 */
3378 	u32 mvfr1;
3379 
3380 	WARN_ON(scope == SCOPE_LOCAL_CPU && preemptible());
3381 	if (scope == SCOPE_SYSTEM)
3382 		mvfr1 = read_sanitised_ftr_reg(SYS_MVFR1_EL1);
3383 	else
3384 		mvfr1 = read_sysreg_s(SYS_MVFR1_EL1);
3385 
3386 	return cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_EL1_SIMDSP_SHIFT) &&
3387 		cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_EL1_SIMDInt_SHIFT) &&
3388 		cpuid_feature_extract_unsigned_field(mvfr1, MVFR1_EL1_SIMDLS_SHIFT);
3389 }
3390 #endif
3391 
3392 static const struct arm64_cpu_capabilities compat_elf_hwcaps[] = {
3393 #ifdef CONFIG_COMPAT
3394 	HWCAP_CAP_MATCH(compat_has_neon, CAP_COMPAT_HWCAP, COMPAT_HWCAP_NEON),
3395 	HWCAP_CAP(MVFR1_EL1, SIMDFMAC, IMP, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFPv4),
3396 	/* Arm v8 mandates MVFR0.FPDP == {0, 2}. So, piggy back on this for the presence of VFP support */
3397 	HWCAP_CAP(MVFR0_EL1, FPDP, VFPv3, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFP),
3398 	HWCAP_CAP(MVFR0_EL1, FPDP, VFPv3, CAP_COMPAT_HWCAP, COMPAT_HWCAP_VFPv3),
3399 	HWCAP_CAP(MVFR1_EL1, FPHP, FP16, CAP_COMPAT_HWCAP, COMPAT_HWCAP_FPHP),
3400 	HWCAP_CAP(MVFR1_EL1, SIMDHP, SIMDHP_FLOAT, CAP_COMPAT_HWCAP, COMPAT_HWCAP_ASIMDHP),
3401 	HWCAP_CAP(ID_ISAR5_EL1, AES, VMULL, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_PMULL),
3402 	HWCAP_CAP(ID_ISAR5_EL1, AES, IMP, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_AES),
3403 	HWCAP_CAP(ID_ISAR5_EL1, SHA1, IMP, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA1),
3404 	HWCAP_CAP(ID_ISAR5_EL1, SHA2, IMP, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SHA2),
3405 	HWCAP_CAP(ID_ISAR5_EL1, CRC32, IMP, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_CRC32),
3406 	HWCAP_CAP(ID_ISAR6_EL1, DP, IMP, CAP_COMPAT_HWCAP, COMPAT_HWCAP_ASIMDDP),
3407 	HWCAP_CAP(ID_ISAR6_EL1, FHM, IMP, CAP_COMPAT_HWCAP, COMPAT_HWCAP_ASIMDFHM),
3408 	HWCAP_CAP(ID_ISAR6_EL1, SB, IMP, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SB),
3409 	HWCAP_CAP(ID_ISAR6_EL1, BF16, IMP, CAP_COMPAT_HWCAP, COMPAT_HWCAP_ASIMDBF16),
3410 	HWCAP_CAP(ID_ISAR6_EL1, I8MM, IMP, CAP_COMPAT_HWCAP, COMPAT_HWCAP_I8MM),
3411 	HWCAP_CAP(ID_PFR2_EL1, SSBS, IMP, CAP_COMPAT_HWCAP2, COMPAT_HWCAP2_SSBS),
3412 #endif
3413 	{},
3414 };
3415 
cap_set_elf_hwcap(const struct arm64_cpu_capabilities * cap)3416 static void cap_set_elf_hwcap(const struct arm64_cpu_capabilities *cap)
3417 {
3418 	switch (cap->hwcap_type) {
3419 	case CAP_HWCAP:
3420 		cpu_set_feature(cap->hwcap);
3421 		break;
3422 #ifdef CONFIG_COMPAT
3423 	case CAP_COMPAT_HWCAP:
3424 		compat_elf_hwcap |= (u32)cap->hwcap;
3425 		break;
3426 	case CAP_COMPAT_HWCAP2:
3427 		compat_elf_hwcap2 |= (u32)cap->hwcap;
3428 		break;
3429 #endif
3430 	default:
3431 		WARN_ON(1);
3432 		break;
3433 	}
3434 }
3435 
3436 /* Check if we have a particular HWCAP enabled */
cpus_have_elf_hwcap(const struct arm64_cpu_capabilities * cap)3437 static bool cpus_have_elf_hwcap(const struct arm64_cpu_capabilities *cap)
3438 {
3439 	bool rc;
3440 
3441 	switch (cap->hwcap_type) {
3442 	case CAP_HWCAP:
3443 		rc = cpu_have_feature(cap->hwcap);
3444 		break;
3445 #ifdef CONFIG_COMPAT
3446 	case CAP_COMPAT_HWCAP:
3447 		rc = (compat_elf_hwcap & (u32)cap->hwcap) != 0;
3448 		break;
3449 	case CAP_COMPAT_HWCAP2:
3450 		rc = (compat_elf_hwcap2 & (u32)cap->hwcap) != 0;
3451 		break;
3452 #endif
3453 	default:
3454 		WARN_ON(1);
3455 		rc = false;
3456 	}
3457 
3458 	return rc;
3459 }
3460 
setup_elf_hwcaps(const struct arm64_cpu_capabilities * hwcaps)3461 static void setup_elf_hwcaps(const struct arm64_cpu_capabilities *hwcaps)
3462 {
3463 	/* We support emulation of accesses to CPU ID feature registers */
3464 	cpu_set_named_feature(CPUID);
3465 	for (; hwcaps->matches; hwcaps++)
3466 		if (hwcaps->matches(hwcaps, cpucap_default_scope(hwcaps)))
3467 			cap_set_elf_hwcap(hwcaps);
3468 }
3469 
update_cpu_capabilities(u16 scope_mask)3470 static void update_cpu_capabilities(u16 scope_mask)
3471 {
3472 	int i;
3473 	const struct arm64_cpu_capabilities *caps;
3474 
3475 	scope_mask &= ARM64_CPUCAP_SCOPE_MASK;
3476 	for (i = 0; i < ARM64_NCAPS; i++) {
3477 		bool match_all = false;
3478 		bool caps_set = false;
3479 		bool boot_cpu = false;
3480 
3481 		caps = cpucap_ptrs[i];
3482 		if (!caps || !(caps->type & scope_mask))
3483 			continue;
3484 
3485 		match_all = cpucap_match_all_early_cpus(caps);
3486 		caps_set = cpus_have_cap(caps->capability);
3487 		boot_cpu = scope_mask & SCOPE_BOOT_CPU;
3488 
3489 		/*
3490 		 * Unless it's a match-all CPUs feature, avoid probing if
3491 		 * already detected.
3492 		 */
3493 		if (!match_all && caps_set)
3494 			continue;
3495 
3496 		/*
3497 		 * A match-all CPUs capability is only set when probing the
3498 		 * boot CPU. It may be cleared subsequently if not detected on
3499 		 * secondary ones.
3500 		 */
3501 		if (match_all && !caps_set && !boot_cpu)
3502 			continue;
3503 
3504 		if (!caps->matches(caps, cpucap_default_scope(caps))) {
3505 			if (match_all)
3506 				__clear_bit(caps->capability, system_cpucaps);
3507 			continue;
3508 		}
3509 
3510 		/*
3511 		 * Match-all CPUs capabilities are logged later when the
3512 		 * system capabilities are finalised.
3513 		 */
3514 		if (!match_all && caps->desc && !caps->cpus)
3515 			pr_info("detected: %s\n", caps->desc);
3516 
3517 		__set_bit(caps->capability, system_cpucaps);
3518 
3519 		if (boot_cpu && (caps->type & SCOPE_BOOT_CPU))
3520 			set_bit(caps->capability, boot_cpucaps);
3521 	}
3522 }
3523 
3524 /*
3525  * Enable all the available capabilities on this CPU. The capabilities
3526  * with BOOT_CPU scope are handled separately and hence skipped here.
3527  */
cpu_enable_non_boot_scope_capabilities(void * __unused)3528 static int cpu_enable_non_boot_scope_capabilities(void *__unused)
3529 {
3530 	int i;
3531 	u16 non_boot_scope = SCOPE_ALL & ~SCOPE_BOOT_CPU;
3532 
3533 	for_each_available_cap(i) {
3534 		const struct arm64_cpu_capabilities *cap = cpucap_ptrs[i];
3535 
3536 		if (WARN_ON(!cap))
3537 			continue;
3538 
3539 		if (!(cap->type & non_boot_scope))
3540 			continue;
3541 
3542 		if (cap->cpu_enable)
3543 			cap->cpu_enable(cap);
3544 	}
3545 	return 0;
3546 }
3547 
3548 /*
3549  * Run through the enabled capabilities and enable() it on all active
3550  * CPUs
3551  */
enable_cpu_capabilities(u16 scope_mask)3552 static void __init enable_cpu_capabilities(u16 scope_mask)
3553 {
3554 	int i;
3555 	const struct arm64_cpu_capabilities *caps;
3556 	bool boot_scope;
3557 
3558 	scope_mask &= ARM64_CPUCAP_SCOPE_MASK;
3559 	boot_scope = !!(scope_mask & SCOPE_BOOT_CPU);
3560 
3561 	for (i = 0; i < ARM64_NCAPS; i++) {
3562 		caps = cpucap_ptrs[i];
3563 		if (!caps || !(caps->type & scope_mask) ||
3564 		    !cpus_have_cap(caps->capability))
3565 			continue;
3566 
3567 		if (boot_scope && caps->cpu_enable)
3568 			/*
3569 			 * Capabilities with SCOPE_BOOT_CPU scope are finalised
3570 			 * before any secondary CPU boots. Thus, each secondary
3571 			 * will enable the capability as appropriate via
3572 			 * check_local_cpu_capabilities(). The only exception is
3573 			 * the boot CPU, for which the capability must be
3574 			 * enabled here. This approach avoids costly
3575 			 * stop_machine() calls for this case.
3576 			 */
3577 			caps->cpu_enable(caps);
3578 	}
3579 
3580 	/*
3581 	 * For all non-boot scope capabilities, use stop_machine()
3582 	 * as it schedules the work allowing us to modify PSTATE,
3583 	 * instead of on_each_cpu() which uses an IPI, giving us a
3584 	 * PSTATE that disappears when we return.
3585 	 */
3586 	if (!boot_scope)
3587 		stop_machine(cpu_enable_non_boot_scope_capabilities,
3588 			     NULL, cpu_online_mask);
3589 }
3590 
3591 /*
3592  * Run through the list of capabilities to check for conflicts.
3593  * If the system has already detected a capability, take necessary
3594  * action on this CPU.
3595  */
verify_local_cpu_caps(u16 scope_mask)3596 static void verify_local_cpu_caps(u16 scope_mask)
3597 {
3598 	int i;
3599 	bool cpu_has_cap, system_has_cap;
3600 	const struct arm64_cpu_capabilities *caps;
3601 
3602 	scope_mask &= ARM64_CPUCAP_SCOPE_MASK;
3603 
3604 	for (i = 0; i < ARM64_NCAPS; i++) {
3605 		caps = cpucap_ptrs[i];
3606 		if (!caps || !(caps->type & scope_mask))
3607 			continue;
3608 
3609 		cpu_has_cap = caps->matches(caps, SCOPE_LOCAL_CPU);
3610 		system_has_cap = cpus_have_cap(caps->capability);
3611 
3612 		if (system_has_cap) {
3613 			/*
3614 			 * Check if the new CPU misses an advertised feature,
3615 			 * which is not safe to miss.
3616 			 */
3617 			if (!cpu_has_cap && !cpucap_late_cpu_optional(caps))
3618 				break;
3619 			/*
3620 			 * We have to issue cpu_enable() irrespective of
3621 			 * whether the CPU has it or not, as it is enabeld
3622 			 * system wide. It is upto the call back to take
3623 			 * appropriate action on this CPU.
3624 			 */
3625 			if (caps->cpu_enable)
3626 				caps->cpu_enable(caps);
3627 		} else {
3628 			/*
3629 			 * Check if the CPU has this capability if it isn't
3630 			 * safe to have when the system doesn't.
3631 			 */
3632 			if (cpu_has_cap && !cpucap_late_cpu_permitted(caps))
3633 				break;
3634 		}
3635 	}
3636 
3637 	if (i < ARM64_NCAPS) {
3638 		pr_crit("CPU%d: Detected conflict for capability %d (%s), System: %d, CPU: %d\n",
3639 			smp_processor_id(), caps->capability,
3640 			caps->desc, system_has_cap, cpu_has_cap);
3641 
3642 		if (cpucap_panic_on_conflict(caps))
3643 			cpu_panic_kernel();
3644 		else
3645 			cpu_die_early();
3646 	}
3647 }
3648 
3649 /*
3650  * Check for CPU features that are used in early boot
3651  * based on the Boot CPU value.
3652  */
check_early_cpu_features(void)3653 static void check_early_cpu_features(void)
3654 {
3655 	verify_cpu_asid_bits();
3656 
3657 	verify_local_cpu_caps(SCOPE_BOOT_CPU);
3658 }
3659 
3660 static void
__verify_local_elf_hwcaps(const struct arm64_cpu_capabilities * caps)3661 __verify_local_elf_hwcaps(const struct arm64_cpu_capabilities *caps)
3662 {
3663 
3664 	for (; caps->matches; caps++)
3665 		if (cpus_have_elf_hwcap(caps) && !caps->matches(caps, SCOPE_LOCAL_CPU)) {
3666 			pr_crit("CPU%d: missing HWCAP: %s\n",
3667 					smp_processor_id(), caps->desc);
3668 			cpu_die_early();
3669 		}
3670 }
3671 
verify_local_elf_hwcaps(void)3672 static void verify_local_elf_hwcaps(void)
3673 {
3674 	__verify_local_elf_hwcaps(arm64_elf_hwcaps);
3675 
3676 	if (id_aa64pfr0_32bit_el0(read_cpuid(ID_AA64PFR0_EL1)))
3677 		__verify_local_elf_hwcaps(compat_elf_hwcaps);
3678 }
3679 
verify_sve_features(void)3680 static void verify_sve_features(void)
3681 {
3682 	unsigned long cpacr = cpacr_save_enable_kernel_sve();
3683 
3684 	if (vec_verify_vq_map(ARM64_VEC_SVE)) {
3685 		pr_crit("CPU%d: SVE: vector length support mismatch\n",
3686 			smp_processor_id());
3687 		cpu_die_early();
3688 	}
3689 
3690 	cpacr_restore(cpacr);
3691 }
3692 
verify_sme_features(void)3693 static void verify_sme_features(void)
3694 {
3695 	unsigned long cpacr = cpacr_save_enable_kernel_sme();
3696 
3697 	if (vec_verify_vq_map(ARM64_VEC_SME)) {
3698 		pr_crit("CPU%d: SME: vector length support mismatch\n",
3699 			smp_processor_id());
3700 		cpu_die_early();
3701 	}
3702 
3703 	cpacr_restore(cpacr);
3704 }
3705 
verify_hyp_capabilities(void)3706 static void verify_hyp_capabilities(void)
3707 {
3708 	u64 safe_mmfr1, mmfr0, mmfr1;
3709 	int parange, ipa_max;
3710 	unsigned int safe_vmid_bits, vmid_bits;
3711 
3712 	if (!IS_ENABLED(CONFIG_KVM))
3713 		return;
3714 
3715 	safe_mmfr1 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR1_EL1);
3716 	mmfr0 = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
3717 	mmfr1 = read_cpuid(ID_AA64MMFR1_EL1);
3718 
3719 	/* Verify VMID bits */
3720 	safe_vmid_bits = get_vmid_bits(safe_mmfr1);
3721 	vmid_bits = get_vmid_bits(mmfr1);
3722 	if (vmid_bits < safe_vmid_bits) {
3723 		pr_crit("CPU%d: VMID width mismatch\n", smp_processor_id());
3724 		cpu_die_early();
3725 	}
3726 
3727 	/* Verify IPA range */
3728 	parange = cpuid_feature_extract_unsigned_field(mmfr0,
3729 				ID_AA64MMFR0_EL1_PARANGE_SHIFT);
3730 	ipa_max = id_aa64mmfr0_parange_to_phys_shift(parange);
3731 	if (ipa_max < get_kvm_ipa_limit()) {
3732 		pr_crit("CPU%d: IPA range mismatch\n", smp_processor_id());
3733 		cpu_die_early();
3734 	}
3735 }
3736 
verify_mpam_capabilities(void)3737 static void verify_mpam_capabilities(void)
3738 {
3739 	u64 cpu_idr = read_cpuid(ID_AA64PFR0_EL1);
3740 	u64 sys_idr = read_sanitised_ftr_reg(SYS_ID_AA64PFR0_EL1);
3741 	u16 cpu_partid_max, cpu_pmg_max, sys_partid_max, sys_pmg_max;
3742 
3743 	if (FIELD_GET(ID_AA64PFR0_EL1_MPAM_MASK, cpu_idr) !=
3744 	    FIELD_GET(ID_AA64PFR0_EL1_MPAM_MASK, sys_idr)) {
3745 		pr_crit("CPU%d: MPAM version mismatch\n", smp_processor_id());
3746 		cpu_die_early();
3747 	}
3748 
3749 	cpu_idr = read_cpuid(MPAMIDR_EL1);
3750 	sys_idr = read_sanitised_ftr_reg(SYS_MPAMIDR_EL1);
3751 	if (FIELD_GET(MPAMIDR_EL1_HAS_HCR, cpu_idr) !=
3752 	    FIELD_GET(MPAMIDR_EL1_HAS_HCR, sys_idr)) {
3753 		pr_crit("CPU%d: Missing MPAM HCR\n", smp_processor_id());
3754 		cpu_die_early();
3755 	}
3756 
3757 	cpu_partid_max = FIELD_GET(MPAMIDR_EL1_PARTID_MAX, cpu_idr);
3758 	cpu_pmg_max = FIELD_GET(MPAMIDR_EL1_PMG_MAX, cpu_idr);
3759 	sys_partid_max = FIELD_GET(MPAMIDR_EL1_PARTID_MAX, sys_idr);
3760 	sys_pmg_max = FIELD_GET(MPAMIDR_EL1_PMG_MAX, sys_idr);
3761 	if (cpu_partid_max < sys_partid_max || cpu_pmg_max < sys_pmg_max) {
3762 		pr_crit("CPU%d: MPAM PARTID/PMG max values are mismatched\n", smp_processor_id());
3763 		cpu_die_early();
3764 	}
3765 }
3766 
3767 /*
3768  * Run through the enabled system capabilities and enable() it on this CPU.
3769  * The capabilities were decided based on the available CPUs at the boot time.
3770  * Any new CPU should match the system wide status of the capability. If the
3771  * new CPU doesn't have a capability which the system now has enabled, we
3772  * cannot do anything to fix it up and could cause unexpected failures. So
3773  * we park the CPU.
3774  */
verify_local_cpu_capabilities(void)3775 static void verify_local_cpu_capabilities(void)
3776 {
3777 	/*
3778 	 * The capabilities with SCOPE_BOOT_CPU are checked from
3779 	 * check_early_cpu_features(), as they need to be verified
3780 	 * on all secondary CPUs.
3781 	 */
3782 	verify_local_cpu_caps(SCOPE_ALL & ~SCOPE_BOOT_CPU);
3783 	verify_local_elf_hwcaps();
3784 
3785 	if (system_supports_sve())
3786 		verify_sve_features();
3787 
3788 	if (system_supports_sme())
3789 		verify_sme_features();
3790 
3791 	if (is_hyp_mode_available())
3792 		verify_hyp_capabilities();
3793 
3794 	if (system_supports_mpam())
3795 		verify_mpam_capabilities();
3796 }
3797 
check_local_cpu_capabilities(void)3798 void check_local_cpu_capabilities(void)
3799 {
3800 	/*
3801 	 * All secondary CPUs should conform to the early CPU features
3802 	 * in use by the kernel based on boot CPU.
3803 	 */
3804 	check_early_cpu_features();
3805 
3806 	/*
3807 	 * If we haven't finalised the system capabilities, this CPU gets
3808 	 * a chance to update the errata work arounds and local features.
3809 	 * Otherwise, this CPU should verify that it has all the system
3810 	 * advertised capabilities.
3811 	 */
3812 	if (!system_capabilities_finalized())
3813 		update_cpu_capabilities(SCOPE_LOCAL_CPU);
3814 	else
3815 		verify_local_cpu_capabilities();
3816 }
3817 
this_cpu_has_cap(unsigned int n)3818 bool this_cpu_has_cap(unsigned int n)
3819 {
3820 	if (!WARN_ON(preemptible()) && n < ARM64_NCAPS) {
3821 		const struct arm64_cpu_capabilities *cap = cpucap_ptrs[n];
3822 
3823 		if (cap)
3824 			return cap->matches(cap, SCOPE_LOCAL_CPU);
3825 	}
3826 
3827 	return false;
3828 }
3829 EXPORT_SYMBOL_GPL(this_cpu_has_cap);
3830 
3831 /*
3832  * This helper function is used in a narrow window when,
3833  * - The system wide safe registers are set with all the SMP CPUs and,
3834  * - The SYSTEM_FEATURE system_cpucaps may not have been set.
3835  */
__system_matches_cap(unsigned int n)3836 static bool __maybe_unused __system_matches_cap(unsigned int n)
3837 {
3838 	if (n < ARM64_NCAPS) {
3839 		const struct arm64_cpu_capabilities *cap = cpucap_ptrs[n];
3840 
3841 		if (cap)
3842 			return cap->matches(cap, SCOPE_SYSTEM);
3843 	}
3844 	return false;
3845 }
3846 
cpu_set_feature(unsigned int num)3847 void cpu_set_feature(unsigned int num)
3848 {
3849 	set_bit(num, elf_hwcap);
3850 }
3851 
cpu_have_feature(unsigned int num)3852 bool cpu_have_feature(unsigned int num)
3853 {
3854 	return test_bit(num, elf_hwcap);
3855 }
3856 EXPORT_SYMBOL_GPL(cpu_have_feature);
3857 
cpu_get_elf_hwcap(void)3858 unsigned long cpu_get_elf_hwcap(void)
3859 {
3860 	/*
3861 	 * We currently only populate the first 32 bits of AT_HWCAP. Please
3862 	 * note that for userspace compatibility we guarantee that bits 62
3863 	 * and 63 will always be returned as 0.
3864 	 */
3865 	return elf_hwcap[0];
3866 }
3867 
cpu_get_elf_hwcap2(void)3868 unsigned long cpu_get_elf_hwcap2(void)
3869 {
3870 	return elf_hwcap[1];
3871 }
3872 
cpu_get_elf_hwcap3(void)3873 unsigned long cpu_get_elf_hwcap3(void)
3874 {
3875 	return elf_hwcap[2];
3876 }
3877 
setup_boot_cpu_capabilities(void)3878 static void __init setup_boot_cpu_capabilities(void)
3879 {
3880 	kvm_arm_target_impl_cpu_init();
3881 	/*
3882 	 * The boot CPU's feature register values have been recorded. Detect
3883 	 * boot cpucaps and local cpucaps for the boot CPU, then enable and
3884 	 * patch alternatives for the available boot cpucaps.
3885 	 */
3886 	update_cpu_capabilities(SCOPE_BOOT_CPU | SCOPE_LOCAL_CPU);
3887 	enable_cpu_capabilities(SCOPE_BOOT_CPU);
3888 	apply_boot_alternatives();
3889 }
3890 
setup_boot_cpu_features(void)3891 void __init setup_boot_cpu_features(void)
3892 {
3893 	/*
3894 	 * Initialize the indirect array of CPU capabilities pointers before we
3895 	 * handle the boot CPU.
3896 	 */
3897 	init_cpucap_indirect_list();
3898 
3899 	/*
3900 	 * Detect broken pseudo-NMI. Must be called _before_ the call to
3901 	 * setup_boot_cpu_capabilities() since it interacts with
3902 	 * can_use_gic_priorities().
3903 	 */
3904 	detect_system_supports_pseudo_nmi();
3905 
3906 	setup_boot_cpu_capabilities();
3907 }
3908 
setup_system_capabilities(void)3909 static void __init setup_system_capabilities(void)
3910 {
3911 	/*
3912 	 * The system-wide safe feature register values have been finalized.
3913 	 * Detect, enable, and patch alternatives for the available system
3914 	 * cpucaps.
3915 	 */
3916 	update_cpu_capabilities(SCOPE_SYSTEM);
3917 	enable_cpu_capabilities(SCOPE_ALL & ~SCOPE_BOOT_CPU);
3918 	apply_alternatives_all();
3919 
3920 	for (int i = 0; i < ARM64_NCAPS; i++) {
3921 		const struct arm64_cpu_capabilities *caps = cpucap_ptrs[i];
3922 
3923 		if (!caps || !caps->desc)
3924 			continue;
3925 
3926 		/*
3927 		 * Log any cpucaps with a cpumask as these aren't logged by
3928 		 * update_cpu_capabilities().
3929 		 */
3930 		if (caps->cpus && cpumask_any(caps->cpus) < nr_cpu_ids)
3931 			pr_info("detected: %s on CPU%*pbl\n",
3932 				caps->desc, cpumask_pr_args(caps->cpus));
3933 
3934 		/* Log match-all CPUs capabilities */
3935 		if (cpucap_match_all_early_cpus(caps) &&
3936 		    cpus_have_cap(caps->capability))
3937 			pr_info("detected: %s\n", caps->desc);
3938 	}
3939 
3940 	/*
3941 	 * TTBR0 PAN doesn't have its own cpucap, so log it manually.
3942 	 */
3943 	if (system_uses_ttbr0_pan())
3944 		pr_info("emulated: Privileged Access Never (PAN) using TTBR0_EL1 switching\n");
3945 }
3946 
setup_system_features(void)3947 void __init setup_system_features(void)
3948 {
3949 	setup_system_capabilities();
3950 
3951 	kpti_install_ng_mappings();
3952 
3953 	sve_setup();
3954 	sme_setup();
3955 
3956 	/*
3957 	 * Check for sane CTR_EL0.CWG value.
3958 	 */
3959 	if (!cache_type_cwg())
3960 		pr_warn("No Cache Writeback Granule information, assuming %d\n",
3961 			ARCH_DMA_MINALIGN);
3962 }
3963 
setup_user_features(void)3964 void __init setup_user_features(void)
3965 {
3966 	user_feature_fixup();
3967 
3968 	setup_elf_hwcaps(arm64_elf_hwcaps);
3969 
3970 	if (system_supports_32bit_el0()) {
3971 		setup_elf_hwcaps(compat_elf_hwcaps);
3972 		elf_hwcap_fixup();
3973 	}
3974 
3975 	minsigstksz_setup();
3976 }
3977 
enable_mismatched_32bit_el0(unsigned int cpu)3978 static int enable_mismatched_32bit_el0(unsigned int cpu)
3979 {
3980 	/*
3981 	 * The first 32-bit-capable CPU we detected and so can no longer
3982 	 * be offlined by userspace. -1 indicates we haven't yet onlined
3983 	 * a 32-bit-capable CPU.
3984 	 */
3985 	static int lucky_winner = -1;
3986 
3987 	struct cpuinfo_arm64 *info = &per_cpu(cpu_data, cpu);
3988 	bool cpu_32bit = false;
3989 
3990 	if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) {
3991 		if (!housekeeping_cpu(cpu, HK_TYPE_TICK))
3992 			pr_info("Treating adaptive-ticks CPU %u as 64-bit only\n", cpu);
3993 		else
3994 			cpu_32bit = true;
3995 	}
3996 
3997 	if (cpu_32bit) {
3998 		cpumask_set_cpu(cpu, cpu_32bit_el0_mask);
3999 		static_branch_enable_cpuslocked(&arm64_mismatched_32bit_el0);
4000 	}
4001 
4002 	if (cpumask_test_cpu(0, cpu_32bit_el0_mask) == cpu_32bit)
4003 		return 0;
4004 
4005 	if (lucky_winner >= 0)
4006 		return 0;
4007 
4008 	/*
4009 	 * We've detected a mismatch. We need to keep one of our CPUs with
4010 	 * 32-bit EL0 online so that is_cpu_allowed() doesn't end up rejecting
4011 	 * every CPU in the system for a 32-bit task.
4012 	 */
4013 	lucky_winner = cpu_32bit ? cpu : cpumask_any_and(cpu_32bit_el0_mask,
4014 							 cpu_active_mask);
4015 	get_cpu_device(lucky_winner)->offline_disabled = true;
4016 	setup_elf_hwcaps(compat_elf_hwcaps);
4017 	elf_hwcap_fixup();
4018 	pr_info("Asymmetric 32-bit EL0 support detected on CPU %u; CPU hot-unplug disabled on CPU %u\n",
4019 		cpu, lucky_winner);
4020 	return 0;
4021 }
4022 
init_32bit_el0_mask(void)4023 static int __init init_32bit_el0_mask(void)
4024 {
4025 	if (!allow_mismatched_32bit_el0)
4026 		return 0;
4027 
4028 	if (!zalloc_cpumask_var(&cpu_32bit_el0_mask, GFP_KERNEL))
4029 		return -ENOMEM;
4030 
4031 	return cpuhp_setup_state(CPUHP_AP_ONLINE_DYN,
4032 				 "arm64/mismatched_32bit_el0:online",
4033 				 enable_mismatched_32bit_el0, NULL);
4034 }
4035 subsys_initcall_sync(init_32bit_el0_mask);
4036 
cpu_enable_cnp(struct arm64_cpu_capabilities const * cap)4037 static void __maybe_unused cpu_enable_cnp(struct arm64_cpu_capabilities const *cap)
4038 {
4039 	cpu_enable_swapper_cnp();
4040 }
4041 
4042 /*
4043  * We emulate only the following system register space.
4044  * Op0 = 0x3, CRn = 0x0, Op1 = 0x0, CRm = [0, 2 - 7]
4045  * See Table C5-6 System instruction encodings for System register accesses,
4046  * ARMv8 ARM(ARM DDI 0487A.f) for more details.
4047  */
is_emulated(u32 id)4048 static inline bool __attribute_const__ is_emulated(u32 id)
4049 {
4050 	return (sys_reg_Op0(id) == 0x3 &&
4051 		sys_reg_CRn(id) == 0x0 &&
4052 		sys_reg_Op1(id) == 0x0 &&
4053 		(sys_reg_CRm(id) == 0 ||
4054 		 ((sys_reg_CRm(id) >= 2) && (sys_reg_CRm(id) <= 7))));
4055 }
4056 
4057 /*
4058  * With CRm == 0, reg should be one of :
4059  * MIDR_EL1, MPIDR_EL1 or REVIDR_EL1.
4060  */
emulate_id_reg(u32 id,u64 * valp)4061 static inline int emulate_id_reg(u32 id, u64 *valp)
4062 {
4063 	switch (id) {
4064 	case SYS_MIDR_EL1:
4065 		*valp = read_cpuid_id();
4066 		break;
4067 	case SYS_MPIDR_EL1:
4068 		*valp = SYS_MPIDR_SAFE_VAL;
4069 		break;
4070 	case SYS_REVIDR_EL1:
4071 		/* IMPLEMENTATION DEFINED values are emulated with 0 */
4072 		*valp = 0;
4073 		break;
4074 	default:
4075 		return -EINVAL;
4076 	}
4077 
4078 	return 0;
4079 }
4080 
emulate_sys_reg(u32 id,u64 * valp)4081 static int emulate_sys_reg(u32 id, u64 *valp)
4082 {
4083 	struct arm64_ftr_reg *regp;
4084 
4085 	if (!is_emulated(id))
4086 		return -EINVAL;
4087 
4088 	if (sys_reg_CRm(id) == 0)
4089 		return emulate_id_reg(id, valp);
4090 
4091 	regp = get_arm64_ftr_reg_nowarn(id);
4092 	if (regp)
4093 		*valp = arm64_ftr_reg_user_value(regp);
4094 	else
4095 		/*
4096 		 * The untracked registers are either IMPLEMENTATION DEFINED
4097 		 * (e.g, ID_AFR0_EL1) or reserved RAZ.
4098 		 */
4099 		*valp = 0;
4100 	return 0;
4101 }
4102 
do_emulate_mrs(struct pt_regs * regs,u32 sys_reg,u32 rt)4103 int do_emulate_mrs(struct pt_regs *regs, u32 sys_reg, u32 rt)
4104 {
4105 	int rc;
4106 	u64 val;
4107 
4108 	rc = emulate_sys_reg(sys_reg, &val);
4109 	if (!rc) {
4110 		pt_regs_write_reg(regs, rt, val);
4111 		arm64_skip_faulting_instruction(regs, AARCH64_INSN_SIZE);
4112 	}
4113 	return rc;
4114 }
4115 
try_emulate_mrs(struct pt_regs * regs,u32 insn)4116 bool try_emulate_mrs(struct pt_regs *regs, u32 insn)
4117 {
4118 	u32 sys_reg, rt;
4119 
4120 	if (compat_user_mode(regs) || !aarch64_insn_is_mrs(insn))
4121 		return false;
4122 
4123 	/*
4124 	 * sys_reg values are defined as used in mrs/msr instruction.
4125 	 * shift the imm value to get the encoding.
4126 	 */
4127 	sys_reg = (u32)aarch64_insn_decode_immediate(AARCH64_INSN_IMM_16, insn) << 5;
4128 	rt = aarch64_insn_decode_register(AARCH64_INSN_REGTYPE_RT, insn);
4129 	return do_emulate_mrs(regs, sys_reg, rt) == 0;
4130 }
4131 
arm64_get_meltdown_state(void)4132 enum mitigation_state arm64_get_meltdown_state(void)
4133 {
4134 	if (__meltdown_safe)
4135 		return SPECTRE_UNAFFECTED;
4136 
4137 	if (arm64_kernel_unmapped_at_el0())
4138 		return SPECTRE_MITIGATED;
4139 
4140 	return SPECTRE_VULNERABLE;
4141 }
4142 
cpu_show_meltdown(struct device * dev,struct device_attribute * attr,char * buf)4143 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr,
4144 			  char *buf)
4145 {
4146 	switch (arm64_get_meltdown_state()) {
4147 	case SPECTRE_UNAFFECTED:
4148 		return sprintf(buf, "Not affected\n");
4149 
4150 	case SPECTRE_MITIGATED:
4151 		return sprintf(buf, "Mitigation: PTI\n");
4152 
4153 	default:
4154 		return sprintf(buf, "Vulnerable\n");
4155 	}
4156 }
4157