1 /*-
2 * Copyright (c) 2014 Andrew Turner
3 * Copyright (c) 2014 The FreeBSD Foundation
4 * All rights reserved.
5 *
6 * Portions of this software were developed by Semihalf
7 * under sponsorship of the FreeBSD Foundation.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 */
31
32 #include <sys/param.h>
33 #include <sys/kernel.h>
34 #include <sys/malloc.h>
35 #include <sys/proc.h>
36 #include <sys/pcpu.h>
37 #include <sys/sbuf.h>
38 #include <sys/smp.h>
39 #include <sys/sysctl.h>
40 #include <sys/sysent.h>
41 #include <sys/systm.h>
42
43 #include <machine/atomic.h>
44 #include <machine/cpu.h>
45 #include <machine/cpu_feat.h>
46 #include <machine/cpufunc.h>
47 #include <machine/elf.h>
48 #include <machine/md_var.h>
49 #include <machine/undefined.h>
50
51 static MALLOC_DEFINE(M_IDENTCPU, "CPU ID", "arm64 CPU identification memory");
52
53 struct cpu_desc;
54
55 static void print_cpu_midr(struct sbuf *sb, u_int cpu);
56 static void print_cpu_features(u_int cpu, struct cpu_desc *desc,
57 struct cpu_desc *prev_desc);
58 static void print_cpu_caches(struct sbuf *sb, struct cpu_desc *desc);
59 #ifdef COMPAT_FREEBSD32
60 static u_long parse_cpu_features_hwcap32(void);
61 #endif
62
63 const char machine[] = "arm64";
64
65 #ifdef SCTL_MASK32
66 extern int adaptive_machine_arch;
67 #endif
68
69 static SYSCTL_NODE(_machdep, OID_AUTO, cache, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
70 "Cache management tuning");
71
72 static int allow_dic = 1;
73 SYSCTL_INT(_machdep_cache, OID_AUTO, allow_dic, CTLFLAG_RDTUN, &allow_dic, 0,
74 "Allow optimizations based on the DIC cache bit");
75
76 static int allow_idc = 1;
77 SYSCTL_INT(_machdep_cache, OID_AUTO, allow_idc, CTLFLAG_RDTUN, &allow_idc, 0,
78 "Allow optimizations based on the IDC cache bit");
79
80 static bool emulate_ctr = false;
81
82 static void check_cpu_regs(u_int cpu, struct cpu_desc *desc,
83 struct cpu_desc *prev_desc);
84 static uint64_t update_special_reg_field(uint64_t user_reg, u_int type,
85 uint64_t value, u_int width, u_int shift, bool sign);
86
87 /*
88 * The default implementation of I-cache sync assumes we have an
89 * aliasing cache until we know otherwise.
90 */
91 void (*arm64_icache_sync_range)(void *, vm_size_t) =
92 &arm64_aliasing_icache_sync_range;
93
94 static int
sysctl_hw_machine(SYSCTL_HANDLER_ARGS)95 sysctl_hw_machine(SYSCTL_HANDLER_ARGS)
96 {
97 #ifdef SCTL_MASK32
98 static const char machine32[] = "arm";
99 #endif
100 int error;
101
102 #ifdef SCTL_MASK32
103 if ((req->flags & SCTL_MASK32) != 0 && adaptive_machine_arch)
104 error = SYSCTL_OUT(req, machine32, sizeof(machine32));
105 else
106 #endif
107 error = SYSCTL_OUT(req, machine, sizeof(machine));
108 return (error);
109 }
110
111 SYSCTL_PROC(_hw, HW_MACHINE, machine, CTLTYPE_STRING | CTLFLAG_RD |
112 CTLFLAG_CAPRD | CTLFLAG_MPSAFE, NULL, 0, sysctl_hw_machine, "A",
113 "Machine class");
114
115 static char cpu_model[64];
116 SYSCTL_STRING(_hw, HW_MODEL, model, CTLFLAG_RD | CTLFLAG_CAPRD,
117 cpu_model, sizeof(cpu_model), "Machine model");
118
119 #define MAX_CACHES 8 /* Maximum number of caches supported
120 architecturally. */
121 /*
122 * Per-CPU affinity as provided in MPIDR_EL1
123 * Indexed by CPU number in logical order selected by the system.
124 * Relevant fields can be extracted using CPU_AFFn macros,
125 * Aff3.Aff2.Aff1.Aff0 construct a unique CPU address in the system.
126 *
127 * Fields used by us:
128 * Aff1 - Cluster number
129 * Aff0 - CPU number in Aff1 cluster
130 */
131 uint64_t __cpu_affinity[MAXCPU];
132 static u_int cpu_aff_levels;
133
134 struct cpu_desc {
135 uint64_t mpidr;
136 uint64_t id_aa64afr0;
137 uint64_t id_aa64afr1;
138 uint64_t id_aa64dfr0;
139 uint64_t id_aa64dfr1;
140 uint64_t id_aa64isar0;
141 uint64_t id_aa64isar1;
142 uint64_t id_aa64isar2;
143 uint64_t id_aa64mmfr0;
144 uint64_t id_aa64mmfr1;
145 uint64_t id_aa64mmfr2;
146 uint64_t id_aa64mmfr3;
147 uint64_t id_aa64mmfr4;
148 uint64_t id_aa64pfr0;
149 uint64_t id_aa64pfr1;
150 uint64_t id_aa64pfr2;
151 uint64_t id_aa64zfr0;
152 uint64_t ctr;
153 #ifdef COMPAT_FREEBSD32
154 uint64_t id_isar5;
155 uint64_t mvfr0;
156 uint64_t mvfr1;
157 #endif
158 uint64_t clidr;
159 uint32_t ccsidr[MAX_CACHES][2]; /* 2 possible types. */
160 bool have_sve;
161 };
162
163 static struct cpu_desc cpu_desc0;
164 static struct cpu_desc *cpu_desc;
165 static struct cpu_desc kern_cpu_desc;
166 static struct cpu_desc user_cpu_desc;
167 static struct cpu_desc l_user_cpu_desc;
168
169 static struct cpu_desc *
get_cpu_desc(u_int cpu)170 get_cpu_desc(u_int cpu)
171 {
172 /* The cpu_desc for CPU 0 is used before the allocator is ready. */
173 if (cpu == 0)
174 return (&cpu_desc0);
175
176 MPASS(cpu_desc != NULL);
177 return (&cpu_desc[cpu - 1]);
178 }
179
180 struct cpu_parts {
181 u_int part_id;
182 const char *part_name;
183 };
184 #define CPU_PART_NONE { 0, NULL }
185
186 struct cpu_implementers {
187 u_int impl_id;
188 const char *impl_name;
189 /*
190 * Part number is implementation defined
191 * so each vendor will have its own set of values and names.
192 */
193 const struct cpu_parts *cpu_parts;
194 };
195 #define CPU_IMPLEMENTER_NONE { 0, NULL, NULL }
196
197 /*
198 * Per-implementer table of (PartNum, CPU Name) pairs.
199 */
200 /* ARM Ltd. */
201 static const struct cpu_parts cpu_parts_arm[] = {
202 { CPU_PART_AEM_V8, "AEMv8" },
203 { CPU_PART_FOUNDATION, "Foundation-Model" },
204 { CPU_PART_CORTEX_A34, "Cortex-A34" },
205 { CPU_PART_CORTEX_A35, "Cortex-A35" },
206 { CPU_PART_CORTEX_A53, "Cortex-A53" },
207 { CPU_PART_CORTEX_A55, "Cortex-A55" },
208 { CPU_PART_CORTEX_A57, "Cortex-A57" },
209 { CPU_PART_CORTEX_A65, "Cortex-A65" },
210 { CPU_PART_CORTEX_A65AE, "Cortex-A65AE" },
211 { CPU_PART_CORTEX_A72, "Cortex-A72" },
212 { CPU_PART_CORTEX_A73, "Cortex-A73" },
213 { CPU_PART_CORTEX_A75, "Cortex-A75" },
214 { CPU_PART_CORTEX_A76, "Cortex-A76" },
215 { CPU_PART_CORTEX_A76AE, "Cortex-A76AE" },
216 { CPU_PART_CORTEX_A77, "Cortex-A77" },
217 { CPU_PART_CORTEX_A78, "Cortex-A78" },
218 { CPU_PART_CORTEX_A78C, "Cortex-A78C" },
219 { CPU_PART_CORTEX_A510, "Cortex-A510" },
220 { CPU_PART_CORTEX_A710, "Cortex-A710" },
221 { CPU_PART_CORTEX_A715, "Cortex-A715" },
222 { CPU_PART_CORTEX_X1, "Cortex-X1" },
223 { CPU_PART_CORTEX_X1C, "Cortex-X1C" },
224 { CPU_PART_CORTEX_X2, "Cortex-X2" },
225 { CPU_PART_CORTEX_X3, "Cortex-X3" },
226 { CPU_PART_NEOVERSE_E1, "Neoverse-E1" },
227 { CPU_PART_NEOVERSE_N1, "Neoverse-N1" },
228 { CPU_PART_NEOVERSE_N2, "Neoverse-N2" },
229 { CPU_PART_NEOVERSE_V1, "Neoverse-V1" },
230 { CPU_PART_NEOVERSE_V2, "Neoverse-V2" },
231 CPU_PART_NONE,
232 };
233
234 /* Cavium */
235 static const struct cpu_parts cpu_parts_cavium[] = {
236 { CPU_PART_THUNDERX, "ThunderX" },
237 { CPU_PART_THUNDERX2, "ThunderX2" },
238 CPU_PART_NONE,
239 };
240
241 /* APM / Ampere */
242 static const struct cpu_parts cpu_parts_apm[] = {
243 { CPU_PART_EMAG8180, "eMAG 8180" },
244 CPU_PART_NONE,
245 };
246
247 /* Qualcomm */
248 static const struct cpu_parts cpu_parts_qcom[] = {
249 { CPU_PART_KRYO400_GOLD, "Kryo 400 Gold" },
250 { CPU_PART_KRYO400_SILVER, "Kryo 400 Silver" },
251 CPU_PART_NONE,
252 };
253
254 /* Apple */
255 static const struct cpu_parts cpu_parts_apple[] = {
256 { CPU_PART_M1_ICESTORM, "M1 Icestorm" },
257 { CPU_PART_M1_FIRESTORM, "M1 Firestorm" },
258 { CPU_PART_M1_ICESTORM_PRO, "M1 Pro Icestorm" },
259 { CPU_PART_M1_FIRESTORM_PRO, "M1 Pro Firestorm" },
260 { CPU_PART_M1_ICESTORM_MAX, "M1 Max Icestorm" },
261 { CPU_PART_M1_FIRESTORM_MAX, "M1 Max Firestorm" },
262 { CPU_PART_M2_BLIZZARD, "M2 Blizzard" },
263 { CPU_PART_M2_AVALANCHE, "M2 Avalanche" },
264 { CPU_PART_M2_BLIZZARD_PRO, "M2 Pro Blizzard" },
265 { CPU_PART_M2_AVALANCHE_PRO, "M2 Pro Avalanche" },
266 { CPU_PART_M2_BLIZZARD_MAX, "M2 Max Blizzard" },
267 { CPU_PART_M2_AVALANCHE_MAX, "M2 Max Avalanche" },
268 CPU_PART_NONE,
269 };
270
271 /* Unknown */
272 static const struct cpu_parts cpu_parts_none[] = {
273 CPU_PART_NONE,
274 };
275
276 /*
277 * Implementers table.
278 */
279 const struct cpu_implementers cpu_implementers[] = {
280 { CPU_IMPL_AMPERE, "Ampere", cpu_parts_none },
281 { CPU_IMPL_APPLE, "Apple", cpu_parts_apple },
282 { CPU_IMPL_APM, "APM", cpu_parts_apm },
283 { CPU_IMPL_ARM, "ARM", cpu_parts_arm },
284 { CPU_IMPL_BROADCOM, "Broadcom", cpu_parts_none },
285 { CPU_IMPL_CAVIUM, "Cavium", cpu_parts_cavium },
286 { CPU_IMPL_DEC, "DEC", cpu_parts_none },
287 { CPU_IMPL_FREESCALE, "Freescale", cpu_parts_none },
288 { CPU_IMPL_FUJITSU, "Fujitsu", cpu_parts_none },
289 { CPU_IMPL_INFINEON, "IFX", cpu_parts_none },
290 { CPU_IMPL_INTEL, "Intel", cpu_parts_none },
291 { CPU_IMPL_MARVELL, "Marvell", cpu_parts_none },
292 { CPU_IMPL_NVIDIA, "NVIDIA", cpu_parts_none },
293 { CPU_IMPL_QUALCOMM, "Qualcomm", cpu_parts_qcom },
294 CPU_IMPLEMENTER_NONE,
295 };
296
297 #define MRS_TYPE_MASK 0xf
298 #define MRS_INVALID 0
299 #define MRS_EXACT 1
300 #define MRS_EXACT_IF_DIFFERENT 2
301 #define MRS_LOWER 3
302 #define MRS_HIGHER_OR_ZERO 4
303 #define MRS_HIGHER 5
304 #define MRS_SAFE_SHIFT 4
305 #define MRS_SAFE_MASK (0xfu << MRS_SAFE_SHIFT)
306 #define MRS_SAFE(x) (((x) << MRS_SAFE_SHIFT) & MRS_SAFE_MASK)
307 #define MRS_SAFE_VAL(x) (((x) & MRS_SAFE_MASK) >> MRS_SAFE_SHIFT)
308 #define MRS_FREEBSD (1u << 8)
309 #define MRS_LINUX (1u << 9)
310 #define MRS_USERSPACE (MRS_FREEBSD | MRS_LINUX)
311
312 struct mrs_field_value {
313 uint64_t value;
314 const char *desc;
315 };
316
317 #define MRS_FIELD_VALUE(_value, _desc) \
318 { \
319 .value = (_value), \
320 .desc = (_desc), \
321 }
322
323 #define MRS_FIELD_VALUE_NONE_IMPL(_reg, _field, _none, _impl) \
324 MRS_FIELD_VALUE(_reg ## _ ## _field ## _ ## _none, ""), \
325 MRS_FIELD_VALUE(_reg ## _ ## _field ## _ ## _impl, #_field)
326
327 #define MRS_FIELD_VALUE_COUNT(_reg, _field, _desc) \
328 MRS_FIELD_VALUE(0ul << _reg ## _ ## _field ## _SHIFT, "1 " _desc), \
329 MRS_FIELD_VALUE(1ul << _reg ## _ ## _field ## _SHIFT, "2 " _desc "s"), \
330 MRS_FIELD_VALUE(2ul << _reg ## _ ## _field ## _SHIFT, "3 " _desc "s"), \
331 MRS_FIELD_VALUE(3ul << _reg ## _ ## _field ## _SHIFT, "4 " _desc "s"), \
332 MRS_FIELD_VALUE(4ul << _reg ## _ ## _field ## _SHIFT, "5 " _desc "s"), \
333 MRS_FIELD_VALUE(5ul << _reg ## _ ## _field ## _SHIFT, "6 " _desc "s"), \
334 MRS_FIELD_VALUE(6ul << _reg ## _ ## _field ## _SHIFT, "7 " _desc "s"), \
335 MRS_FIELD_VALUE(7ul << _reg ## _ ## _field ## _SHIFT, "8 " _desc "s"), \
336 MRS_FIELD_VALUE(8ul << _reg ## _ ## _field ## _SHIFT, "9 " _desc "s"), \
337 MRS_FIELD_VALUE(9ul << _reg ## _ ## _field ## _SHIFT, "10 "_desc "s"), \
338 MRS_FIELD_VALUE(10ul<< _reg ## _ ## _field ## _SHIFT, "11 "_desc "s"), \
339 MRS_FIELD_VALUE(11ul<< _reg ## _ ## _field ## _SHIFT, "12 "_desc "s"), \
340 MRS_FIELD_VALUE(12ul<< _reg ## _ ## _field ## _SHIFT, "13 "_desc "s"), \
341 MRS_FIELD_VALUE(13ul<< _reg ## _ ## _field ## _SHIFT, "14 "_desc "s"), \
342 MRS_FIELD_VALUE(14ul<< _reg ## _ ## _field ## _SHIFT, "15 "_desc "s"), \
343 MRS_FIELD_VALUE(15ul<< _reg ## _ ## _field ## _SHIFT, "16 "_desc "s")
344
345 /*
346 * Used for printing I/D cache line sizes & CWG/ERG, as 0 is a special case
347 * in some cases the decoded string needs to be passed in.
348 */
349 #define MRS_FIELD_VALUE_CACHE(_reg, _field, _0desc, _desc) \
350 MRS_FIELD_VALUE(0ul << _reg ## _ ## _field ## _SHIFT, _0desc), \
351 MRS_FIELD_VALUE(1ul << _reg ## _ ## _field ## _SHIFT, "8 " _desc), \
352 MRS_FIELD_VALUE(2ul << _reg ## _ ## _field ## _SHIFT, "16 " _desc), \
353 MRS_FIELD_VALUE(3ul << _reg ## _ ## _field ## _SHIFT, "32 " _desc), \
354 MRS_FIELD_VALUE(4ul << _reg ## _ ## _field ## _SHIFT, "64 " _desc), \
355 MRS_FIELD_VALUE(5ul << _reg ## _ ## _field ## _SHIFT, "128 " _desc), \
356 MRS_FIELD_VALUE(6ul << _reg ## _ ## _field ## _SHIFT, "256 " _desc), \
357 MRS_FIELD_VALUE(7ul << _reg ## _ ## _field ## _SHIFT, "512 " _desc), \
358 MRS_FIELD_VALUE(8ul << _reg ## _ ## _field ## _SHIFT, "1k " _desc), \
359 MRS_FIELD_VALUE(9ul << _reg ## _ ## _field ## _SHIFT, "2k " _desc), \
360 MRS_FIELD_VALUE(10ul<< _reg ## _ ## _field ## _SHIFT, "4k " _desc), \
361 MRS_FIELD_VALUE(11ul<< _reg ## _ ## _field ## _SHIFT, "8k " _desc), \
362 MRS_FIELD_VALUE(12ul<< _reg ## _ ## _field ## _SHIFT, "16k " _desc), \
363 MRS_FIELD_VALUE(13ul<< _reg ## _ ## _field ## _SHIFT, "32k " _desc), \
364 MRS_FIELD_VALUE(14ul<< _reg ## _ ## _field ## _SHIFT, "64k " _desc), \
365 MRS_FIELD_VALUE(15ul<< _reg ## _ ## _field ## _SHIFT, "128k "_desc)
366
367 #define MRS_FIELD_VALUE_END { .desc = NULL }
368
369 struct mrs_field_hwcap {
370 uint64_t min;
371 u_long hwcap_val;
372 u_int hwcap_id;
373 };
374
375 #define MRS_HWCAP(_hwcap_id, _val, _min) \
376 { \
377 .hwcap_id = (_hwcap_id), \
378 .hwcap_val = (_val), \
379 .min = (_min), \
380 }
381
382 #define MRS_HWCAP_END { .hwcap_id = 0 }
383
384 struct mrs_field {
385 const char *name;
386 const struct mrs_field_value *values;
387 const struct mrs_field_hwcap *hwcaps;
388 uint64_t mask;
389 bool sign;
390 u_int type;
391 u_int width;
392 u_int shift;
393 };
394
395 #define MRS_FIELD_RES1(_width, _shift) \
396 { \
397 .sign = false, \
398 .type = MRS_EXACT | MRS_SAFE((1u << (_width)) - 1) | \
399 MRS_USERSPACE, \
400 .width = (_width), \
401 .shift = (_shift), \
402 }
403
404 #define MRS_FIELD_HWCAP(_register, _name, _sign, _type, _visibility, \
405 _values, _hwcap) \
406 { \
407 .name = #_name, \
408 .sign = (_sign), \
409 .type = ((_type) | (_visibility)), \
410 .width = _register ## _ ## _name ## _WIDTH, \
411 .shift = _register ## _ ## _name ## _SHIFT, \
412 .mask = _register ## _ ## _name ## _MASK, \
413 .values = (_values), \
414 .hwcaps = (_hwcap), \
415 }
416
417 #define MRS_FIELD(_register, _name, _sign, _type, _visibility, _values) \
418 MRS_FIELD_HWCAP(_register, _name, _sign, _type, _visibility, \
419 _values, NULL)
420
421 #define MRS_FIELD_END { .type = MRS_INVALID, }
422
423 /* CTR_EL0 */
424 static const struct mrs_field_value ctr_dic[] = {
425 MRS_FIELD_VALUE_NONE_IMPL(CTR, DIC, NONE, IMPL),
426 MRS_FIELD_VALUE_END,
427 };
428
429 static const struct mrs_field_value ctr_idc[] = {
430 MRS_FIELD_VALUE_NONE_IMPL(CTR, IDC, NONE, IMPL),
431 MRS_FIELD_VALUE_END,
432 };
433
434 static const struct mrs_field_value ctr_cwg[] = {
435 MRS_FIELD_VALUE_CACHE(CTR, CWG, "Unknown CWG",
436 "byte CWG"),
437 MRS_FIELD_VALUE_END,
438 };
439
440 static const struct mrs_field_value ctr_erg[] = {
441 MRS_FIELD_VALUE_CACHE(CTR, ERG, "Unknown ERG",
442 "byte ERG"),
443 MRS_FIELD_VALUE_END,
444 };
445
446 static const struct mrs_field_value ctr_dline[] = {
447 MRS_FIELD_VALUE_CACHE(CTR, DLINE, "4 byte D-cacheline",
448 "byte D-cacheline"),
449 MRS_FIELD_VALUE_END,
450 };
451
452 static const struct mrs_field_value ctr_l1ip[] = {
453 MRS_FIELD_VALUE(CTR_L1IP_VIPT, "VIPT I-cache"),
454 MRS_FIELD_VALUE(CTR_L1IP_PIPT, "PIPT I-cache"),
455 MRS_FIELD_VALUE_END,
456 };
457
458 static const struct mrs_field_value ctr_iline[] = {
459 MRS_FIELD_VALUE_CACHE(CTR, ILINE, "4 byte I-cacheline",
460 "byte I-cacheline"),
461 MRS_FIELD_VALUE_END,
462 };
463
464 static const struct mrs_field ctr_fields[] = {
465 /* Bit 31 is RES1 */
466 MRS_FIELD_RES1(1, 31),
467 MRS_FIELD(CTR, DIC, false, MRS_LOWER, MRS_USERSPACE, ctr_dic),
468 MRS_FIELD(CTR, IDC, false, MRS_LOWER, MRS_USERSPACE, ctr_idc),
469 MRS_FIELD(CTR, CWG, false, MRS_HIGHER_OR_ZERO, MRS_USERSPACE, ctr_cwg),
470 MRS_FIELD(CTR, ERG, false, MRS_HIGHER_OR_ZERO, MRS_USERSPACE, ctr_erg),
471 MRS_FIELD(CTR, DLINE, false, MRS_LOWER, MRS_USERSPACE, ctr_dline),
472 /* If the ICache types are different report the safe option */
473 MRS_FIELD(CTR, L1IP, false, MRS_EXACT_IF_DIFFERENT |
474 MRS_SAFE(CTR_L1IP_VIPT >> CTR_L1IP_SHIFT), MRS_USERSPACE,
475 ctr_l1ip),
476 MRS_FIELD(CTR, ILINE, false, MRS_LOWER, MRS_USERSPACE, ctr_iline),
477 MRS_FIELD_END,
478 };
479
480 /* ID_AA64AFR0_EL1 */
481 static const struct mrs_field id_aa64afr0_fields[] = {
482 MRS_FIELD_END,
483 };
484
485
486 /* ID_AA64AFR1_EL1 */
487 static const struct mrs_field id_aa64afr1_fields[] = {
488 MRS_FIELD_END,
489 };
490
491
492 /* ID_AA64DFR0_EL1 */
493 static const struct mrs_field_value id_aa64dfr0_hpmn0[] = {
494 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR0, HPMN0, CONSTR, DEFINED),
495 MRS_FIELD_VALUE_END,
496 };
497
498 static const struct mrs_field_value id_aa64dfr0_brbe[] = {
499 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR0, BRBE, NONE, IMPL),
500 MRS_FIELD_VALUE(ID_AA64DFR0_BRBE_EL3, "BRBE EL3"),
501 MRS_FIELD_VALUE_END,
502 };
503
504 static const struct mrs_field_value id_aa64dfr0_mtpmu[] = {
505 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR0, MTPMU, NONE, IMPL),
506 MRS_FIELD_VALUE(ID_AA64DFR0_MTPMU_NONE_MT_RES0, "MTPMU res0"),
507 MRS_FIELD_VALUE_END,
508 };
509
510 static const struct mrs_field_value id_aa64dfr0_tracebuffer[] = {
511 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR0, TraceBuffer, NONE, IMPL),
512 MRS_FIELD_VALUE_END,
513 };
514
515 static const struct mrs_field_value id_aa64dfr0_tracefilt[] = {
516 MRS_FIELD_VALUE(ID_AA64DFR0_TraceFilt_NONE, ""),
517 MRS_FIELD_VALUE(ID_AA64DFR0_TraceFilt_8_4, "Trace v8.4"),
518 MRS_FIELD_VALUE_END,
519 };
520
521 static const struct mrs_field_value id_aa64dfr0_doublelock[] = {
522 MRS_FIELD_VALUE(ID_AA64DFR0_DoubleLock_IMPL, "DoubleLock"),
523 MRS_FIELD_VALUE(ID_AA64DFR0_DoubleLock_NONE, ""),
524 MRS_FIELD_VALUE_END,
525 };
526
527 static const struct mrs_field_value id_aa64dfr0_pmsver[] = {
528 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_NONE, ""),
529 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE, "SPE"),
530 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE_1_1, "SPEv1p1"),
531 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE_1_2, "SPEv1p2"),
532 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE_1_3, "SPEv1p3"),
533 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE_1_4, "SPEv1p4"),
534 MRS_FIELD_VALUE_END,
535 };
536
537 static const struct mrs_field_value id_aa64dfr0_ctx_cmps[] = {
538 MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, CTX_CMPs, "CTX BKPT"),
539 MRS_FIELD_VALUE_END,
540 };
541
542 static const struct mrs_field_value id_aa64dfr0_wrps[] = {
543 MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, WRPs, "Watchpoint"),
544 MRS_FIELD_VALUE_END,
545 };
546
547 static const struct mrs_field_value id_aa64dfr0_pmss[] = {
548 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR0, PMSS, NONE, IMPL),
549 MRS_FIELD_VALUE_END,
550 };
551
552 static const struct mrs_field_value id_aa64dfr0_brps[] = {
553 MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, BRPs, "Breakpoint"),
554 MRS_FIELD_VALUE_END,
555 };
556
557 static const struct mrs_field_value id_aa64dfr0_pmuver[] = {
558 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_NONE, ""),
559 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3, "PMUv3"),
560 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_1, "PMUv3p1"),
561 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_4, "PMUv3p4"),
562 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_5, "PMUv3p5"),
563 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_7, "PMUv3p7"),
564 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_8, "PMUv3p8"),
565 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_9, "PMUv3p9"),
566 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_IMPL, "IMPL PMU"),
567 MRS_FIELD_VALUE_END,
568 };
569
570 static const struct mrs_field_value id_aa64dfr0_tracever[] = {
571 MRS_FIELD_VALUE(ID_AA64DFR0_TraceVer_NONE, ""),
572 MRS_FIELD_VALUE(ID_AA64DFR0_TraceVer_IMPL, "Trace"),
573 MRS_FIELD_VALUE_END,
574 };
575
576 static const struct mrs_field_value id_aa64dfr0_debugver[] = {
577 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8, "Debugv8"),
578 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_VHE, "Debugv8_VHE"),
579 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_2, "Debugv8p2"),
580 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_4, "Debugv8p4"),
581 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_8, "Debugv8p8"),
582 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_9, "Debugv8p9"),
583 MRS_FIELD_VALUE_END,
584 };
585
586 static const struct mrs_field id_aa64dfr0_fields[] = {
587 MRS_FIELD(ID_AA64DFR0, HPMN0, false, MRS_LOWER, 0, id_aa64dfr0_hpmn0),
588 MRS_FIELD(ID_AA64DFR0, BRBE, false, MRS_LOWER, 0, id_aa64dfr0_brbe),
589 MRS_FIELD(ID_AA64DFR0, MTPMU, true, MRS_LOWER, 0, id_aa64dfr0_mtpmu),
590 MRS_FIELD(ID_AA64DFR0, TraceBuffer, false, MRS_LOWER, 0,
591 id_aa64dfr0_tracebuffer),
592 MRS_FIELD(ID_AA64DFR0, TraceFilt, false, MRS_LOWER, 0,
593 id_aa64dfr0_tracefilt),
594 MRS_FIELD(ID_AA64DFR0, DoubleLock, false, MRS_LOWER, 0,
595 id_aa64dfr0_doublelock),
596 MRS_FIELD(ID_AA64DFR0, PMSVer, false, MRS_LOWER, 0, id_aa64dfr0_pmsver),
597 MRS_FIELD(ID_AA64DFR0, CTX_CMPs, false, MRS_LOWER, 0,
598 id_aa64dfr0_ctx_cmps),
599 MRS_FIELD(ID_AA64DFR0, WRPs, false, MRS_LOWER, MRS_USERSPACE,
600 id_aa64dfr0_wrps),
601 MRS_FIELD(ID_AA64DFR0, PMSS, false, MRS_LOWER, 0, id_aa64dfr0_pmss),
602 MRS_FIELD(ID_AA64DFR0, BRPs, false, MRS_LOWER, MRS_USERSPACE,
603 id_aa64dfr0_brps),
604 MRS_FIELD(ID_AA64DFR0, PMUVer, false, MRS_LOWER, 0, id_aa64dfr0_pmuver),
605 MRS_FIELD(ID_AA64DFR0, TraceVer, false, MRS_LOWER, 0,
606 id_aa64dfr0_tracever),
607 MRS_FIELD(ID_AA64DFR0, DebugVer, false, MRS_LOWER | MRS_SAFE(0x6), 0,
608 id_aa64dfr0_debugver),
609 MRS_FIELD_END,
610 };
611
612
613 /* ID_AA64DFR1_EL1 */
614 static const struct mrs_field_value id_aa64dfr1_dpfzs[] = {
615 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR1, DPFZS, NONE, IMPL),
616 MRS_FIELD_VALUE_END,
617 };
618
619 static const struct mrs_field_value id_aa64dfr1_pmicntr[] = {
620 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR1, PMICNTR, NONE, IMPL),
621 MRS_FIELD_VALUE_END,
622 };
623
624 static const struct mrs_field_value id_aa64dfr1_spmu[] = {
625 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR1, SPMU, NONE, IMPL),
626 MRS_FIELD_VALUE_END,
627 };
628
629 static const struct mrs_field id_aa64dfr1_fields[] = {
630 MRS_FIELD(ID_AA64DFR1, DPFZS, false, MRS_LOWER, 0, id_aa64dfr1_dpfzs),
631 MRS_FIELD(ID_AA64DFR1, PMICNTR, false, MRS_LOWER, 0, id_aa64dfr1_pmicntr),
632 MRS_FIELD(ID_AA64DFR1, SPMU, false, MRS_LOWER, 0, id_aa64dfr1_spmu),
633 MRS_FIELD_END,
634 };
635
636
637 /* ID_AA64ISAR0_EL1 */
638 static const struct mrs_field_value id_aa64isar0_rndr[] = {
639 MRS_FIELD_VALUE(ID_AA64ISAR0_RNDR_NONE, ""),
640 MRS_FIELD_VALUE(ID_AA64ISAR0_RNDR_IMPL, "RNG"),
641 MRS_FIELD_VALUE_END,
642 };
643
644 static const struct mrs_field_hwcap id_aa64isar0_rndr_caps[] = {
645 MRS_HWCAP(2, HWCAP2_RNG, ID_AA64ISAR0_RNDR_IMPL),
646 MRS_HWCAP_END
647 };
648
649 static const struct mrs_field_value id_aa64isar0_tlb[] = {
650 MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_NONE, ""),
651 MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_TLBIOS, "TLBI-OS"),
652 MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_TLBIOSR, "TLBI-OSR"),
653 MRS_FIELD_VALUE_END,
654 };
655
656 static const struct mrs_field_value id_aa64isar0_ts[] = {
657 MRS_FIELD_VALUE(ID_AA64ISAR0_TS_NONE, ""),
658 MRS_FIELD_VALUE(ID_AA64ISAR0_TS_CondM_8_4, "CondM-8.4"),
659 MRS_FIELD_VALUE(ID_AA64ISAR0_TS_CondM_8_5, "CondM-8.5"),
660 MRS_FIELD_VALUE_END,
661 };
662
663 static const struct mrs_field_hwcap id_aa64isar0_ts_caps[] = {
664 MRS_HWCAP(1, HWCAP_FLAGM, ID_AA64ISAR0_TS_CondM_8_4),
665 MRS_HWCAP(2, HWCAP2_FLAGM2, ID_AA64ISAR0_TS_CondM_8_5),
666 MRS_HWCAP_END
667 };
668
669 static const struct mrs_field_value id_aa64isar0_fhm[] = {
670 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, FHM, NONE, IMPL),
671 MRS_FIELD_VALUE_END,
672 };
673
674 static const struct mrs_field_hwcap id_aa64isar0_fhm_caps[] = {
675 MRS_HWCAP(1, HWCAP_ASIMDFHM, ID_AA64ISAR0_FHM_IMPL),
676 MRS_HWCAP_END
677 };
678
679 static const struct mrs_field_value id_aa64isar0_dp[] = {
680 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, DP, NONE, IMPL),
681 MRS_FIELD_VALUE_END,
682 };
683
684 static const struct mrs_field_hwcap id_aa64isar0_dp_caps[] = {
685 MRS_HWCAP(1, HWCAP_ASIMDDP, ID_AA64ISAR0_DP_IMPL),
686 MRS_HWCAP_END
687 };
688
689 static const struct mrs_field_value id_aa64isar0_sm4[] = {
690 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SM4, NONE, IMPL),
691 MRS_FIELD_VALUE_END,
692 };
693
694 static const struct mrs_field_hwcap id_aa64isar0_sm4_caps[] = {
695 MRS_HWCAP(1, HWCAP_SM4, ID_AA64ISAR0_SM4_IMPL),
696 MRS_HWCAP_END
697 };
698
699 static const struct mrs_field_value id_aa64isar0_sm3[] = {
700 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SM3, NONE, IMPL),
701 MRS_FIELD_VALUE_END,
702 };
703
704 static const struct mrs_field_hwcap id_aa64isar0_sm3_caps[] = {
705 MRS_HWCAP(1, HWCAP_SM3, ID_AA64ISAR0_SM3_IMPL),
706 MRS_HWCAP_END
707 };
708
709 static const struct mrs_field_value id_aa64isar0_sha3[] = {
710 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA3, NONE, IMPL),
711 MRS_FIELD_VALUE_END,
712 };
713
714 static const struct mrs_field_hwcap id_aa64isar0_sha3_caps[] = {
715 MRS_HWCAP(1, HWCAP_SHA3, ID_AA64ISAR0_SHA3_IMPL),
716 MRS_HWCAP_END
717 };
718
719 static const struct mrs_field_value id_aa64isar0_rdm[] = {
720 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, RDM, NONE, IMPL),
721 MRS_FIELD_VALUE_END,
722 };
723
724 static const struct mrs_field_hwcap id_aa64isar0_rdm_caps[] = {
725 MRS_HWCAP(1, HWCAP_ASIMDRDM, ID_AA64ISAR0_RDM_IMPL),
726 MRS_HWCAP_END
727 };
728
729 static const struct mrs_field_value id_aa64isar0_tme[] = {
730 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, TME, NONE, IMPL),
731 MRS_FIELD_VALUE_END,
732 };
733
734 static const struct mrs_field_value id_aa64isar0_atomic[] = {
735 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, Atomic, NONE, IMPL),
736 MRS_FIELD_VALUE_END,
737 };
738
739 static const struct mrs_field_hwcap id_aa64isar0_atomic_caps[] = {
740 MRS_HWCAP(1, HWCAP_ATOMICS, ID_AA64ISAR0_Atomic_IMPL),
741 MRS_HWCAP_END
742 };
743
744 static const struct mrs_field_value id_aa64isar0_crc32[] = {
745 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, CRC32, NONE, BASE),
746 MRS_FIELD_VALUE_END,
747 };
748
749 static const struct mrs_field_hwcap id_aa64isar0_crc32_caps[] = {
750 MRS_HWCAP(1, HWCAP_CRC32, ID_AA64ISAR0_CRC32_BASE),
751 MRS_HWCAP_END
752 };
753
754 static const struct mrs_field_value id_aa64isar0_sha2[] = {
755 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA2, NONE, BASE),
756 MRS_FIELD_VALUE(ID_AA64ISAR0_SHA2_512, "SHA2+SHA512"),
757 MRS_FIELD_VALUE_END,
758 };
759
760 static const struct mrs_field_hwcap id_aa64isar0_sha2_caps[] = {
761 MRS_HWCAP(1, HWCAP_SHA2, ID_AA64ISAR0_SHA2_BASE),
762 MRS_HWCAP(1, HWCAP_SHA512, ID_AA64ISAR0_SHA2_512),
763 MRS_HWCAP_END
764 };
765
766 static const struct mrs_field_value id_aa64isar0_sha1[] = {
767 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA1, NONE, BASE),
768 MRS_FIELD_VALUE_END,
769 };
770
771 static const struct mrs_field_hwcap id_aa64isar0_sha1_caps[] = {
772 MRS_HWCAP(1, HWCAP_SHA1, ID_AA64ISAR0_SHA1_BASE),
773 MRS_HWCAP_END
774 };
775
776 static const struct mrs_field_value id_aa64isar0_aes[] = {
777 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, AES, NONE, BASE),
778 MRS_FIELD_VALUE(ID_AA64ISAR0_AES_PMULL, "AES+PMULL"),
779 MRS_FIELD_VALUE_END,
780 };
781
782 static const struct mrs_field_hwcap id_aa64isar0_aes_caps[] = {
783 MRS_HWCAP(1, HWCAP_AES, ID_AA64ISAR0_AES_BASE),
784 MRS_HWCAP(1, HWCAP_PMULL, ID_AA64ISAR0_AES_PMULL),
785 MRS_HWCAP_END
786 };
787
788 static const struct mrs_field id_aa64isar0_fields[] = {
789 MRS_FIELD_HWCAP(ID_AA64ISAR0, RNDR, false, MRS_LOWER, MRS_USERSPACE,
790 id_aa64isar0_rndr, id_aa64isar0_rndr_caps),
791 MRS_FIELD(ID_AA64ISAR0, TLB, false, MRS_LOWER, 0, id_aa64isar0_tlb),
792 MRS_FIELD_HWCAP(ID_AA64ISAR0, TS, false, MRS_LOWER, MRS_USERSPACE,
793 id_aa64isar0_ts, id_aa64isar0_ts_caps),
794 MRS_FIELD_HWCAP(ID_AA64ISAR0, FHM, false, MRS_LOWER, MRS_USERSPACE,
795 id_aa64isar0_fhm, id_aa64isar0_fhm_caps),
796 MRS_FIELD_HWCAP(ID_AA64ISAR0, DP, false, MRS_LOWER, MRS_USERSPACE,
797 id_aa64isar0_dp, id_aa64isar0_dp_caps),
798 MRS_FIELD_HWCAP(ID_AA64ISAR0, SM4, false, MRS_LOWER, MRS_USERSPACE,
799 id_aa64isar0_sm4, id_aa64isar0_sm4_caps),
800 MRS_FIELD_HWCAP(ID_AA64ISAR0, SM3, false, MRS_LOWER, MRS_USERSPACE,
801 id_aa64isar0_sm3, id_aa64isar0_sm3_caps),
802 MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA3, false, MRS_LOWER, MRS_USERSPACE,
803 id_aa64isar0_sha3, id_aa64isar0_sha3_caps),
804 MRS_FIELD_HWCAP(ID_AA64ISAR0, RDM, false, MRS_LOWER, MRS_USERSPACE,
805 id_aa64isar0_rdm, id_aa64isar0_rdm_caps),
806 MRS_FIELD(ID_AA64ISAR0, TME, false, MRS_LOWER, 0, id_aa64isar0_tme),
807 MRS_FIELD_HWCAP(ID_AA64ISAR0, Atomic, false, MRS_LOWER, MRS_USERSPACE,
808 id_aa64isar0_atomic, id_aa64isar0_atomic_caps),
809 MRS_FIELD_HWCAP(ID_AA64ISAR0, CRC32, false, MRS_LOWER, MRS_USERSPACE,
810 id_aa64isar0_crc32, id_aa64isar0_crc32_caps),
811 MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA2, false, MRS_LOWER, MRS_USERSPACE,
812 id_aa64isar0_sha2, id_aa64isar0_sha2_caps),
813 MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA1, false, MRS_LOWER, MRS_USERSPACE,
814 id_aa64isar0_sha1, id_aa64isar0_sha1_caps),
815 MRS_FIELD_HWCAP(ID_AA64ISAR0, AES, false, MRS_LOWER, MRS_USERSPACE,
816 id_aa64isar0_aes, id_aa64isar0_aes_caps),
817 MRS_FIELD_END,
818 };
819
820
821 /* ID_AA64ISAR1_EL1 */
822 static const struct mrs_field_value id_aa64isar1_ls64[] = {
823 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, LS64, NONE, IMPL),
824 MRS_FIELD_VALUE(ID_AA64ISAR1_LS64_V, "LS64v"),
825 MRS_FIELD_VALUE(ID_AA64ISAR1_LS64_ACCDATA, "LS64+ACCDATA"),
826 MRS_FIELD_VALUE_END,
827 };
828
829 static const struct mrs_field_value id_aa64isar1_xs[] = {
830 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, XS, NONE, IMPL),
831 MRS_FIELD_VALUE_END,
832 };
833
834 static const struct mrs_field_value id_aa64isar1_i8mm[] = {
835 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, I8MM, NONE, IMPL),
836 MRS_FIELD_VALUE_END,
837 };
838
839 static const struct mrs_field_hwcap id_aa64isar1_i8mm_caps[] = {
840 MRS_HWCAP(2, HWCAP2_I8MM, ID_AA64ISAR1_I8MM_IMPL),
841 MRS_HWCAP_END
842 };
843
844 static const struct mrs_field_value id_aa64isar1_dgh[] = {
845 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, DGH, NONE, IMPL),
846 MRS_FIELD_VALUE_END,
847 };
848
849 static const struct mrs_field_hwcap id_aa64isar1_dgh_caps[] = {
850 MRS_HWCAP(2, HWCAP2_DGH, ID_AA64ISAR1_DGH_IMPL),
851 MRS_HWCAP_END
852 };
853
854 static const struct mrs_field_value id_aa64isar1_bf16[] = {
855 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, BF16, NONE, IMPL),
856 MRS_FIELD_VALUE(ID_AA64ISAR1_BF16_EBF, "EBF16"),
857 MRS_FIELD_VALUE_END,
858 };
859
860 static const struct mrs_field_hwcap id_aa64isar1_bf16_caps[] = {
861 MRS_HWCAP(2, HWCAP2_BF16, ID_AA64ISAR1_BF16_IMPL),
862 MRS_HWCAP_END
863 };
864
865 static const struct mrs_field_value id_aa64isar1_specres[] = {
866 MRS_FIELD_VALUE(ID_AA64ISAR1_SPECRES_NONE, ""),
867 MRS_FIELD_VALUE(ID_AA64ISAR1_SPECRES_8_5, "PredInv v8.5"),
868 MRS_FIELD_VALUE(ID_AA64ISAR1_SPECRES_8_9, "PredInv v8.9"),
869 MRS_FIELD_VALUE_END,
870 };
871
872 static const struct mrs_field_value id_aa64isar1_sb[] = {
873 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, SB, NONE, IMPL),
874 MRS_FIELD_VALUE_END,
875 };
876
877 static const struct mrs_field_hwcap id_aa64isar1_sb_caps[] = {
878 MRS_HWCAP(1, HWCAP_SB, ID_AA64ISAR1_SB_IMPL),
879 MRS_HWCAP_END
880 };
881
882 static const struct mrs_field_value id_aa64isar1_frintts[] = {
883 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, FRINTTS, NONE, IMPL),
884 MRS_FIELD_VALUE_END,
885 };
886
887 static const struct mrs_field_hwcap id_aa64isar1_frintts_caps[] = {
888 MRS_HWCAP(2, HWCAP2_FRINT, ID_AA64ISAR1_FRINTTS_IMPL),
889 MRS_HWCAP_END
890 };
891
892 static const struct mrs_field_value id_aa64isar1_gpi[] = {
893 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, GPI, NONE, IMPL),
894 MRS_FIELD_VALUE_END,
895 };
896
897 static const struct mrs_field_hwcap id_aa64isar1_gpi_caps[] = {
898 MRS_HWCAP(1, HWCAP_PACG, ID_AA64ISAR1_GPI_IMPL),
899 MRS_HWCAP_END
900 };
901
902 static const struct mrs_field_value id_aa64isar1_gpa[] = {
903 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, GPA, NONE, IMPL),
904 MRS_FIELD_VALUE_END,
905 };
906
907 static const struct mrs_field_hwcap id_aa64isar1_gpa_caps[] = {
908 MRS_HWCAP(1, HWCAP_PACG, ID_AA64ISAR1_GPA_IMPL),
909 MRS_HWCAP_END
910 };
911
912 static const struct mrs_field_value id_aa64isar1_lrcpc[] = {
913 MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_NONE, ""),
914 MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_RCPC_8_3, "RCPC-8.3"),
915 MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_RCPC_8_4, "RCPC-8.4"),
916 MRS_FIELD_VALUE_END,
917 };
918
919 static const struct mrs_field_hwcap id_aa64isar1_lrcpc_caps[] = {
920 MRS_HWCAP(1, HWCAP_LRCPC, ID_AA64ISAR1_LRCPC_RCPC_8_3),
921 MRS_HWCAP(1, HWCAP_ILRCPC, ID_AA64ISAR1_LRCPC_RCPC_8_4),
922 MRS_HWCAP_END
923 };
924
925 static const struct mrs_field_value id_aa64isar1_fcma[] = {
926 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, FCMA, NONE, IMPL),
927 MRS_FIELD_VALUE_END,
928 };
929
930 static const struct mrs_field_hwcap id_aa64isar1_fcma_caps[] = {
931 MRS_HWCAP(1, HWCAP_FCMA, ID_AA64ISAR1_FCMA_IMPL),
932 MRS_HWCAP_END
933 };
934
935 static const struct mrs_field_value id_aa64isar1_jscvt[] = {
936 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, JSCVT, NONE, IMPL),
937 MRS_FIELD_VALUE_END,
938 };
939
940 static const struct mrs_field_hwcap id_aa64isar1_jscvt_caps[] = {
941 MRS_HWCAP(1, HWCAP_JSCVT, ID_AA64ISAR1_JSCVT_IMPL),
942 MRS_HWCAP_END
943 };
944
945 static const struct mrs_field_value id_aa64isar1_api[] = {
946 MRS_FIELD_VALUE(ID_AA64ISAR1_API_NONE, ""),
947 MRS_FIELD_VALUE(ID_AA64ISAR1_API_PAC, "API PAC"),
948 MRS_FIELD_VALUE(ID_AA64ISAR1_API_EPAC, "API EPAC"),
949 MRS_FIELD_VALUE(ID_AA64ISAR1_API_EPAC2, "Impl PAuth+EPAC2"),
950 MRS_FIELD_VALUE(ID_AA64ISAR1_API_FPAC, "Impl PAuth+FPAC"),
951 MRS_FIELD_VALUE(ID_AA64ISAR1_API_FPAC_COMBINED,
952 "Impl PAuth+FPAC+Combined"),
953 MRS_FIELD_VALUE_END,
954 };
955
956 static const struct mrs_field_hwcap id_aa64isar1_api_caps[] = {
957 MRS_HWCAP(1, HWCAP_PACA, ID_AA64ISAR1_API_PAC),
958 MRS_HWCAP_END
959 };
960
961 static const struct mrs_field_value id_aa64isar1_apa[] = {
962 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_NONE, ""),
963 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_PAC, "APA PAC"),
964 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_EPAC, "APA EPAC"),
965 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_EPAC2, "APA EPAC2"),
966 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_FPAC, "APA FPAC"),
967 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_FPAC_COMBINED,
968 "APA FPAC+Combined"),
969 MRS_FIELD_VALUE_END,
970 };
971
972 static const struct mrs_field_hwcap id_aa64isar1_apa_caps[] = {
973 MRS_HWCAP(1, HWCAP_PACA, ID_AA64ISAR1_APA_PAC),
974 MRS_HWCAP_END
975 };
976
977 static const struct mrs_field_value id_aa64isar1_dpb[] = {
978 MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_NONE, ""),
979 MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_DCCVAP, "DCPoP"),
980 MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_DCCVADP, "DCCVADP"),
981 MRS_FIELD_VALUE_END,
982 };
983
984 static const struct mrs_field_hwcap id_aa64isar1_dpb_caps[] = {
985 MRS_HWCAP(1, HWCAP_DCPOP, ID_AA64ISAR1_DPB_DCCVAP),
986 MRS_HWCAP(2, HWCAP2_DCPODP, ID_AA64ISAR1_DPB_DCCVADP),
987 MRS_HWCAP_END
988 };
989
990 static const struct mrs_field id_aa64isar1_fields[] = {
991 MRS_FIELD(ID_AA64ISAR1, LS64, false, MRS_LOWER, 0, id_aa64isar1_ls64),
992 MRS_FIELD(ID_AA64ISAR1, XS, false, MRS_LOWER, 0, id_aa64isar1_xs),
993 MRS_FIELD_HWCAP(ID_AA64ISAR1, I8MM, false, MRS_LOWER, MRS_USERSPACE,
994 id_aa64isar1_i8mm, id_aa64isar1_i8mm_caps),
995 MRS_FIELD_HWCAP(ID_AA64ISAR1, DGH, false, MRS_LOWER, MRS_USERSPACE,
996 id_aa64isar1_dgh, id_aa64isar1_dgh_caps),
997 MRS_FIELD_HWCAP(ID_AA64ISAR1, BF16, false, MRS_LOWER, MRS_USERSPACE,
998 id_aa64isar1_bf16, id_aa64isar1_bf16_caps),
999 MRS_FIELD(ID_AA64ISAR1, SPECRES, false, MRS_LOWER, 0,
1000 id_aa64isar1_specres),
1001 MRS_FIELD_HWCAP(ID_AA64ISAR1, SB, false, MRS_LOWER, MRS_USERSPACE,
1002 id_aa64isar1_sb, id_aa64isar1_sb_caps),
1003 MRS_FIELD_HWCAP(ID_AA64ISAR1, FRINTTS, false, MRS_LOWER, MRS_USERSPACE,
1004 id_aa64isar1_frintts, id_aa64isar1_frintts_caps),
1005 MRS_FIELD_HWCAP(ID_AA64ISAR1, GPI, false, MRS_LOWER, 0,
1006 id_aa64isar1_gpi, id_aa64isar1_gpi_caps),
1007 MRS_FIELD_HWCAP(ID_AA64ISAR1, GPA, false, MRS_LOWER, 0,
1008 id_aa64isar1_gpa, id_aa64isar1_gpa_caps),
1009 MRS_FIELD_HWCAP(ID_AA64ISAR1, LRCPC, false, MRS_LOWER, MRS_USERSPACE,
1010 id_aa64isar1_lrcpc, id_aa64isar1_lrcpc_caps),
1011 MRS_FIELD_HWCAP(ID_AA64ISAR1, FCMA, false, MRS_LOWER, MRS_USERSPACE,
1012 id_aa64isar1_fcma, id_aa64isar1_fcma_caps),
1013 MRS_FIELD_HWCAP(ID_AA64ISAR1, JSCVT, false, MRS_LOWER, MRS_USERSPACE,
1014 id_aa64isar1_jscvt, id_aa64isar1_jscvt_caps),
1015 MRS_FIELD_HWCAP(ID_AA64ISAR1, API, false, MRS_LOWER, 0,
1016 id_aa64isar1_api, id_aa64isar1_api_caps),
1017 MRS_FIELD_HWCAP(ID_AA64ISAR1, APA, false, MRS_LOWER, 0,
1018 id_aa64isar1_apa, id_aa64isar1_apa_caps),
1019 MRS_FIELD_HWCAP(ID_AA64ISAR1, DPB, false, MRS_LOWER, MRS_USERSPACE,
1020 id_aa64isar1_dpb, id_aa64isar1_dpb_caps),
1021 MRS_FIELD_END,
1022 };
1023
1024
1025 /* ID_AA64ISAR2_EL1 */
1026 static const struct mrs_field_value id_aa64isar2_ats1a[] = {
1027 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, ATS1A, NONE, IMPL),
1028 MRS_FIELD_VALUE_END,
1029 };
1030
1031 static const struct mrs_field_value id_aa64isar2_cssc[] = {
1032 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, CSSC, NONE, IMPL),
1033 MRS_FIELD_VALUE_END,
1034 };
1035
1036 static const struct mrs_field_value id_aa64isar2_rprfm[] = {
1037 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, RPRFM, NONE, IMPL),
1038 MRS_FIELD_VALUE_END,
1039 };
1040
1041 static const struct mrs_field_value id_aa64isar2_prfmslc[] = {
1042 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, PRFMSLC, NONE, IMPL),
1043 MRS_FIELD_VALUE_END,
1044 };
1045
1046 static const struct mrs_field_value id_aa64isar2_clrbhb[] = {
1047 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, CLRBHB, NONE, IMPL),
1048 MRS_FIELD_VALUE_END,
1049 };
1050
1051 static const struct mrs_field_value id_aa64isar2_pac_frac[] = {
1052 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, PAC_frac, NONE, IMPL),
1053 MRS_FIELD_VALUE_END,
1054 };
1055
1056 static const struct mrs_field_value id_aa64isar2_bc[] = {
1057 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, BC, NONE, IMPL),
1058 MRS_FIELD_VALUE_END,
1059 };
1060
1061 static const struct mrs_field_value id_aa64isar2_mops[] = {
1062 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, MOPS, NONE, IMPL),
1063 MRS_FIELD_VALUE_END,
1064 };
1065
1066 static const struct mrs_field_value id_aa64isar2_apa3[] = {
1067 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_NONE, ""),
1068 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_PAC, "APA3 PAC"),
1069 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_EPAC, "APA3 EPAC"),
1070 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_EPAC2, "APA3 EPAC2"),
1071 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_FPAC, "APA3 FPAC"),
1072 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_FPAC_COMBINED,
1073 "APA3 FPAC+Combined"),
1074 MRS_FIELD_VALUE_END,
1075 };
1076
1077 static const struct mrs_field_hwcap id_aa64isar2_apa3_caps[] = {
1078 MRS_HWCAP(1, HWCAP_PACA, ID_AA64ISAR2_APA3_PAC),
1079 MRS_HWCAP_END
1080 };
1081
1082 static const struct mrs_field_value id_aa64isar2_gpa3[] = {
1083 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, GPA3, NONE, IMPL),
1084 MRS_FIELD_VALUE_END,
1085 };
1086
1087 static const struct mrs_field_hwcap id_aa64isar2_gpa3_caps[] = {
1088 MRS_HWCAP(1, HWCAP_PACG, ID_AA64ISAR2_GPA3_IMPL),
1089 MRS_HWCAP_END
1090 };
1091
1092 static const struct mrs_field_value id_aa64isar2_rpres[] = {
1093 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, RPRES, NONE, IMPL),
1094 MRS_FIELD_VALUE_END,
1095 };
1096
1097 static const struct mrs_field_hwcap id_aa64isar2_rpres_caps[] = {
1098 MRS_HWCAP(2, HWCAP2_RPRES, ID_AA64ISAR2_RPRES_IMPL),
1099 MRS_HWCAP_END
1100 };
1101
1102 static const struct mrs_field_value id_aa64isar2_wfxt[] = {
1103 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, WFxT, NONE, IMPL),
1104 MRS_FIELD_VALUE_END,
1105 };
1106
1107 static const struct mrs_field_hwcap id_aa64isar2_wfxt_caps[] = {
1108 MRS_HWCAP(2, HWCAP2_WFXT, ID_AA64ISAR2_WFxT_IMPL),
1109 MRS_HWCAP_END
1110 };
1111
1112 static const struct mrs_field id_aa64isar2_fields[] = {
1113 MRS_FIELD(ID_AA64ISAR2, ATS1A, false, MRS_LOWER, 0, id_aa64isar2_ats1a),
1114 MRS_FIELD(ID_AA64ISAR2, CSSC, false, MRS_LOWER, 0, id_aa64isar2_cssc),
1115 MRS_FIELD(ID_AA64ISAR2, RPRFM, false, MRS_LOWER, 0, id_aa64isar2_rprfm),
1116 MRS_FIELD(ID_AA64ISAR2, PRFMSLC, false, MRS_LOWER, 0, id_aa64isar2_prfmslc),
1117 MRS_FIELD(ID_AA64ISAR2, CLRBHB, false, MRS_LOWER, 0, id_aa64isar2_clrbhb),
1118 MRS_FIELD(ID_AA64ISAR2, PAC_frac, false, MRS_LOWER, 0,
1119 id_aa64isar2_pac_frac),
1120 MRS_FIELD(ID_AA64ISAR2, BC, false, MRS_LOWER, 0, id_aa64isar2_bc),
1121 MRS_FIELD(ID_AA64ISAR2, MOPS, false, MRS_LOWER, 0, id_aa64isar2_mops),
1122 MRS_FIELD_HWCAP(ID_AA64ISAR2, APA3, false, MRS_LOWER, 0,
1123 id_aa64isar2_apa3, id_aa64isar2_apa3_caps),
1124 MRS_FIELD_HWCAP(ID_AA64ISAR2, GPA3, false, MRS_LOWER, 0,
1125 id_aa64isar2_gpa3, id_aa64isar2_gpa3_caps),
1126 MRS_FIELD_HWCAP(ID_AA64ISAR2, RPRES, false, MRS_LOWER, MRS_USERSPACE,
1127 id_aa64isar2_rpres, id_aa64isar2_rpres_caps),
1128 MRS_FIELD_HWCAP(ID_AA64ISAR2, WFxT, false, MRS_LOWER, 0,
1129 id_aa64isar2_wfxt, id_aa64isar2_wfxt_caps),
1130 MRS_FIELD_END,
1131 };
1132
1133
1134 /* ID_AA64MMFR0_EL1 */
1135 static const struct mrs_field_value id_aa64mmfr0_ecv[] = {
1136 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, ECV, NONE, IMPL),
1137 MRS_FIELD_VALUE(ID_AA64MMFR0_ECV_CNTHCTL, "ECV+CNTHCTL"),
1138 MRS_FIELD_VALUE_END,
1139 };
1140
1141 static const struct mrs_field_value id_aa64mmfr0_fgt[] = {
1142 MRS_FIELD_VALUE(ID_AA64MMFR0_FGT_NONE, ""),
1143 MRS_FIELD_VALUE(ID_AA64MMFR0_FGT_8_6, "FGT v8.6"),
1144 MRS_FIELD_VALUE(ID_AA64MMFR0_FGT_8_9, "FGT v8.9"),
1145 MRS_FIELD_VALUE_END,
1146 };
1147
1148 static const struct mrs_field_value id_aa64mmfr0_exs[] = {
1149 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, ExS, ALL, IMPL),
1150 MRS_FIELD_VALUE_END,
1151 };
1152
1153 static const struct mrs_field_value id_aa64mmfr0_tgran4_2[] = {
1154 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_TGran4, ""),
1155 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_NONE, "No S2 TGran4"),
1156 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_IMPL, "S2 TGran4"),
1157 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_LPA2, "S2 TGran4+LPA2"),
1158 MRS_FIELD_VALUE_END,
1159 };
1160
1161 static const struct mrs_field_value id_aa64mmfr0_tgran64_2[] = {
1162 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_TGran64, ""),
1163 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_NONE, "No S2 TGran64"),
1164 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_IMPL, "S2 TGran64"),
1165 MRS_FIELD_VALUE_END,
1166 };
1167
1168 static const struct mrs_field_value id_aa64mmfr0_tgran16_2[] = {
1169 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_TGran16, ""),
1170 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_NONE, "No S2 TGran16"),
1171 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_IMPL, "S2 TGran16"),
1172 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_LPA2, "S2 TGran16+LPA2"),
1173 MRS_FIELD_VALUE_END,
1174 };
1175
1176 static const struct mrs_field_value id_aa64mmfr0_tgran4[] = {
1177 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran4, NONE, IMPL),
1178 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_LPA2, "TGran4+LPA2"),
1179 MRS_FIELD_VALUE_END,
1180 };
1181
1182 static const struct mrs_field_value id_aa64mmfr0_tgran64[] = {
1183 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran64, NONE, IMPL),
1184 MRS_FIELD_VALUE_END,
1185 };
1186
1187 static const struct mrs_field_value id_aa64mmfr0_tgran16[] = {
1188 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran16, NONE, IMPL),
1189 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_LPA2, "TGran16+LPA2"),
1190 MRS_FIELD_VALUE_END,
1191 };
1192
1193 static const struct mrs_field_value id_aa64mmfr0_bigendel0[] = {
1194 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, BigEndEL0, FIXED, MIXED),
1195 MRS_FIELD_VALUE_END,
1196 };
1197
1198 static const struct mrs_field_value id_aa64mmfr0_snsmem[] = {
1199 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, SNSMem, NONE, DISTINCT),
1200 MRS_FIELD_VALUE_END,
1201 };
1202
1203 static const struct mrs_field_value id_aa64mmfr0_bigend[] = {
1204 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, BigEnd, FIXED, MIXED),
1205 MRS_FIELD_VALUE_END,
1206 };
1207
1208 static const struct mrs_field_value id_aa64mmfr0_asidbits[] = {
1209 MRS_FIELD_VALUE(ID_AA64MMFR0_ASIDBits_8, "8bit ASID"),
1210 MRS_FIELD_VALUE(ID_AA64MMFR0_ASIDBits_16, "16bit ASID"),
1211 MRS_FIELD_VALUE_END,
1212 };
1213
1214 static const struct mrs_field_value id_aa64mmfr0_parange[] = {
1215 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4G, "4GB PA"),
1216 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_64G, "64GB PA"),
1217 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_1T, "1TB PA"),
1218 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4T, "4TB PA"),
1219 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_16T, "16TB PA"),
1220 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_256T, "256TB PA"),
1221 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4P, "4PB PA"),
1222 MRS_FIELD_VALUE_END,
1223 };
1224
1225 static const struct mrs_field id_aa64mmfr0_fields[] = {
1226 MRS_FIELD(ID_AA64MMFR0, ECV, false, MRS_LOWER, 0, id_aa64mmfr0_ecv),
1227 MRS_FIELD(ID_AA64MMFR0, FGT, false, MRS_LOWER, 0, id_aa64mmfr0_fgt),
1228 MRS_FIELD(ID_AA64MMFR0, ExS, false, MRS_LOWER, 0, id_aa64mmfr0_exs),
1229 MRS_FIELD(ID_AA64MMFR0, TGran4_2, false, MRS_LOWER, 0,
1230 id_aa64mmfr0_tgran4_2),
1231 MRS_FIELD(ID_AA64MMFR0, TGran64_2, false, MRS_LOWER, 0,
1232 id_aa64mmfr0_tgran64_2),
1233 MRS_FIELD(ID_AA64MMFR0, TGran16_2, false, MRS_LOWER, 0,
1234 id_aa64mmfr0_tgran16_2),
1235 MRS_FIELD(ID_AA64MMFR0, TGran4, false, MRS_LOWER, 0,
1236 id_aa64mmfr0_tgran4),
1237 MRS_FIELD(ID_AA64MMFR0, TGran64, false, MRS_LOWER, 0,
1238 id_aa64mmfr0_tgran64),
1239 MRS_FIELD(ID_AA64MMFR0, TGran16, false, MRS_LOWER, 0,
1240 id_aa64mmfr0_tgran16),
1241 MRS_FIELD(ID_AA64MMFR0, BigEndEL0, false, MRS_LOWER, 0,
1242 id_aa64mmfr0_bigendel0),
1243 MRS_FIELD(ID_AA64MMFR0, SNSMem, false, MRS_LOWER, 0,
1244 id_aa64mmfr0_snsmem),
1245 MRS_FIELD(ID_AA64MMFR0, BigEnd, false, MRS_LOWER, 0,
1246 id_aa64mmfr0_bigend),
1247 MRS_FIELD(ID_AA64MMFR0, ASIDBits, false, MRS_LOWER, 0,
1248 id_aa64mmfr0_asidbits),
1249 MRS_FIELD(ID_AA64MMFR0, PARange, false, MRS_LOWER, 0,
1250 id_aa64mmfr0_parange),
1251 MRS_FIELD_END,
1252 };
1253
1254
1255 /* ID_AA64MMFR1_EL1 */
1256 static const struct mrs_field_value id_aa64mmfr1_ecbhb[] = {
1257 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, ECBHB, NONE, IMPL),
1258 MRS_FIELD_VALUE_END,
1259 };
1260
1261 static const struct mrs_field_value id_aa64mmfr1_cmovw[] = {
1262 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, CMOVW, NONE, IMPL),
1263 MRS_FIELD_VALUE_END,
1264 };
1265
1266 static const struct mrs_field_value id_aa64mmfr1_tidcp1[] = {
1267 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, TIDCP1, NONE, IMPL),
1268 MRS_FIELD_VALUE_END,
1269 };
1270
1271 static const struct mrs_field_value id_aa64mmfr1_ntlbpa[] = {
1272 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, nTLBPA, NONE, IMPL),
1273 MRS_FIELD_VALUE_END,
1274 };
1275
1276 static const struct mrs_field_value id_aa64mmfr1_afp[] = {
1277 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, AFP, NONE, IMPL),
1278 MRS_FIELD_VALUE_END,
1279 };
1280
1281 static const struct mrs_field_hwcap id_aa64mmfr1_afp_caps[] = {
1282 MRS_HWCAP(2, HWCAP2_AFP, ID_AA64MMFR1_AFP_IMPL),
1283 MRS_HWCAP_END
1284 };
1285
1286 static const struct mrs_field_value id_aa64mmfr1_hcx[] = {
1287 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, HCX, NONE, IMPL),
1288 MRS_FIELD_VALUE_END,
1289 };
1290
1291 static const struct mrs_field_value id_aa64mmfr1_ets[] = {
1292 MRS_FIELD_VALUE(ID_AA64MMFR1_ETS_NONE, ""),
1293 MRS_FIELD_VALUE(ID_AA64MMFR1_ETS_NONE2, ""),
1294 MRS_FIELD_VALUE(ID_AA64MMFR1_ETS_IMPL, "ETS2"),
1295 MRS_FIELD_VALUE_END,
1296 };
1297
1298 static const struct mrs_field_value id_aa64mmfr1_twed[] = {
1299 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, TWED, NONE, IMPL),
1300 MRS_FIELD_VALUE_END,
1301 };
1302
1303 static const struct mrs_field_value id_aa64mmfr1_xnx[] = {
1304 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, XNX, NONE, IMPL),
1305 MRS_FIELD_VALUE_END,
1306 };
1307
1308 static const struct mrs_field_value id_aa64mmfr1_specsei[] = {
1309 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, SpecSEI, NONE, IMPL),
1310 MRS_FIELD_VALUE_END,
1311 };
1312
1313 static const struct mrs_field_value id_aa64mmfr1_pan[] = {
1314 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, PAN, NONE, IMPL),
1315 MRS_FIELD_VALUE(ID_AA64MMFR1_PAN_ATS1E1, "PAN+ATS1E1"),
1316 MRS_FIELD_VALUE(ID_AA64MMFR1_PAN_EPAN, "EPAN"),
1317 MRS_FIELD_VALUE_END,
1318 };
1319
1320 static const struct mrs_field_value id_aa64mmfr1_lo[] = {
1321 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, LO, NONE, IMPL),
1322 MRS_FIELD_VALUE_END,
1323 };
1324
1325 static const struct mrs_field_value id_aa64mmfr1_hpds[] = {
1326 MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_NONE, ""),
1327 MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_HPD, "HPD"),
1328 MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_TTPBHA, "HPD+TTPBHA"),
1329 MRS_FIELD_VALUE_END,
1330 };
1331
1332 static const struct mrs_field_value id_aa64mmfr1_vh[] = {
1333 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, VH, NONE, IMPL),
1334 MRS_FIELD_VALUE_END,
1335 };
1336
1337 static const struct mrs_field_value id_aa64mmfr1_vmidbits[] = {
1338 MRS_FIELD_VALUE(ID_AA64MMFR1_VMIDBits_8, "8bit VMID"),
1339 MRS_FIELD_VALUE(ID_AA64MMFR1_VMIDBits_16, "16bit VMID"),
1340 MRS_FIELD_VALUE_END,
1341 };
1342
1343 static const struct mrs_field_value id_aa64mmfr1_hafdbs[] = {
1344 MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_NONE, ""),
1345 MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_AF, "HAF"),
1346 MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_AF_DBS, "HAF+DS"),
1347 MRS_FIELD_VALUE_END,
1348 };
1349
1350 static const struct mrs_field id_aa64mmfr1_fields[] = {
1351 MRS_FIELD(ID_AA64MMFR1, ECBHB, false, MRS_LOWER, 0, id_aa64mmfr1_ecbhb),
1352 MRS_FIELD(ID_AA64MMFR1, CMOVW, false, MRS_LOWER, 0, id_aa64mmfr1_cmovw),
1353 MRS_FIELD(ID_AA64MMFR1, TIDCP1, false, MRS_LOWER, 0,
1354 id_aa64mmfr1_tidcp1),
1355 MRS_FIELD(ID_AA64MMFR1, nTLBPA, false, MRS_LOWER, 0,
1356 id_aa64mmfr1_ntlbpa),
1357 MRS_FIELD_HWCAP(ID_AA64MMFR1, AFP, false, MRS_LOWER, 0,
1358 id_aa64mmfr1_afp, id_aa64mmfr1_afp_caps),
1359 MRS_FIELD(ID_AA64MMFR1, HCX, false, MRS_LOWER, 0, id_aa64mmfr1_hcx),
1360 MRS_FIELD(ID_AA64MMFR1, ETS, false, MRS_LOWER, 0, id_aa64mmfr1_ets),
1361 MRS_FIELD(ID_AA64MMFR1, TWED, false, MRS_LOWER, 0, id_aa64mmfr1_twed),
1362 MRS_FIELD(ID_AA64MMFR1, XNX, false, MRS_LOWER, 0, id_aa64mmfr1_xnx),
1363 /*
1364 * SpecSEI != 0 indicates the CPU might generate an external abort
1365 * under speculation, while 0 indicates it can't happen. It's safer
1366 * to incorrectly indicate it might happen when it can't rather than
1367 * say it can't happen when it could. As such use the largest value
1368 * found in the system.
1369 */
1370 MRS_FIELD(ID_AA64MMFR1, SpecSEI, false, MRS_HIGHER, 0,
1371 id_aa64mmfr1_specsei),
1372 MRS_FIELD(ID_AA64MMFR1, PAN, false, MRS_LOWER, 0, id_aa64mmfr1_pan),
1373 MRS_FIELD(ID_AA64MMFR1, LO, false, MRS_LOWER, 0, id_aa64mmfr1_lo),
1374 MRS_FIELD(ID_AA64MMFR1, HPDS, false, MRS_LOWER, 0, id_aa64mmfr1_hpds),
1375 MRS_FIELD(ID_AA64MMFR1, VH, false, MRS_LOWER, 0, id_aa64mmfr1_vh),
1376 MRS_FIELD(ID_AA64MMFR1, VMIDBits, false, MRS_LOWER, 0,
1377 id_aa64mmfr1_vmidbits),
1378 MRS_FIELD(ID_AA64MMFR1, HAFDBS, false, MRS_LOWER, 0, id_aa64mmfr1_hafdbs),
1379 MRS_FIELD_END,
1380 };
1381
1382
1383 /* ID_AA64MMFR2_EL1 */
1384 static const struct mrs_field_value id_aa64mmfr2_e0pd[] = {
1385 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, E0PD, NONE, IMPL),
1386 MRS_FIELD_VALUE_END,
1387 };
1388
1389 static const struct mrs_field_value id_aa64mmfr2_evt[] = {
1390 MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_NONE, ""),
1391 MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_8_2, "EVT-8.2"),
1392 MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_8_5, "EVT-8.5"),
1393 MRS_FIELD_VALUE_END,
1394 };
1395
1396 static const struct mrs_field_value id_aa64mmfr2_bbm[] = {
1397 MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL0, ""),
1398 MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL1, "BBM level 1"),
1399 MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL2, "BBM level 2"),
1400 MRS_FIELD_VALUE_END,
1401 };
1402
1403 static const struct mrs_field_value id_aa64mmfr2_ttl[] = {
1404 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, TTL, NONE, IMPL),
1405 MRS_FIELD_VALUE_END,
1406 };
1407
1408 static const struct mrs_field_value id_aa64mmfr2_fwb[] = {
1409 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, FWB, NONE, IMPL),
1410 MRS_FIELD_VALUE_END,
1411 };
1412
1413 static const struct mrs_field_value id_aa64mmfr2_ids[] = {
1414 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, IDS, NONE, IMPL),
1415 MRS_FIELD_VALUE_END,
1416 };
1417
1418 static const struct mrs_field_value id_aa64mmfr2_at[] = {
1419 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, AT, NONE, IMPL),
1420 MRS_FIELD_VALUE_END,
1421 };
1422
1423 static const struct mrs_field_hwcap id_aa64mmfr2_at_caps[] = {
1424 MRS_HWCAP(1, HWCAP_USCAT, ID_AA64MMFR2_AT_IMPL),
1425 MRS_HWCAP_END
1426 };
1427
1428 static const struct mrs_field_value id_aa64mmfr2_st[] = {
1429 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, ST, NONE, IMPL),
1430 MRS_FIELD_VALUE_END,
1431 };
1432
1433 static const struct mrs_field_value id_aa64mmfr2_nv[] = {
1434 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, NV, NONE, 8_3),
1435 MRS_FIELD_VALUE(ID_AA64MMFR2_NV_8_4, "NV v8.4"),
1436 MRS_FIELD_VALUE_END,
1437 };
1438
1439 static const struct mrs_field_value id_aa64mmfr2_ccidx[] = {
1440 MRS_FIELD_VALUE(ID_AA64MMFR2_CCIDX_32, "32bit CCIDX"),
1441 MRS_FIELD_VALUE(ID_AA64MMFR2_CCIDX_64, "64bit CCIDX"),
1442 MRS_FIELD_VALUE_END,
1443 };
1444
1445 static const struct mrs_field_value id_aa64mmfr2_varange[] = {
1446 MRS_FIELD_VALUE(ID_AA64MMFR2_VARange_48, "48bit VA"),
1447 MRS_FIELD_VALUE(ID_AA64MMFR2_VARange_52, "52bit VA"),
1448 MRS_FIELD_VALUE_END,
1449 };
1450
1451 static const struct mrs_field_value id_aa64mmfr2_iesb[] = {
1452 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, IESB, NONE, IMPL),
1453 MRS_FIELD_VALUE_END,
1454 };
1455
1456 static const struct mrs_field_value id_aa64mmfr2_lsm[] = {
1457 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, LSM, NONE, IMPL),
1458 MRS_FIELD_VALUE_END,
1459 };
1460
1461 static const struct mrs_field_value id_aa64mmfr2_uao[] = {
1462 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, UAO, NONE, IMPL),
1463 MRS_FIELD_VALUE_END,
1464 };
1465
1466 static const struct mrs_field_value id_aa64mmfr2_cnp[] = {
1467 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, CnP, NONE, IMPL),
1468 MRS_FIELD_VALUE_END,
1469 };
1470
1471 static const struct mrs_field id_aa64mmfr2_fields[] = {
1472 MRS_FIELD(ID_AA64MMFR2, E0PD, false, MRS_LOWER, 0, id_aa64mmfr2_e0pd),
1473 MRS_FIELD(ID_AA64MMFR2, EVT, false, MRS_LOWER, 0, id_aa64mmfr2_evt),
1474 MRS_FIELD(ID_AA64MMFR2, BBM, false, MRS_LOWER, 0, id_aa64mmfr2_bbm),
1475 MRS_FIELD(ID_AA64MMFR2, TTL, false, MRS_LOWER, 0, id_aa64mmfr2_ttl),
1476 MRS_FIELD(ID_AA64MMFR2, FWB, false, MRS_LOWER, 0, id_aa64mmfr2_fwb),
1477 MRS_FIELD(ID_AA64MMFR2, IDS, false, MRS_LOWER, 0, id_aa64mmfr2_ids),
1478 MRS_FIELD_HWCAP(ID_AA64MMFR2, AT, false, MRS_LOWER, MRS_USERSPACE,
1479 id_aa64mmfr2_at, id_aa64mmfr2_at_caps),
1480 MRS_FIELD(ID_AA64MMFR2, ST, false, MRS_LOWER, 0, id_aa64mmfr2_st),
1481 MRS_FIELD(ID_AA64MMFR2, NV, false, MRS_LOWER, 0, id_aa64mmfr2_nv),
1482 MRS_FIELD(ID_AA64MMFR2, CCIDX, false, MRS_LOWER, 0, id_aa64mmfr2_ccidx),
1483 MRS_FIELD(ID_AA64MMFR2, VARange, false, MRS_LOWER, 0,
1484 id_aa64mmfr2_varange),
1485 MRS_FIELD(ID_AA64MMFR2, IESB, false, MRS_LOWER, 0, id_aa64mmfr2_iesb),
1486 MRS_FIELD(ID_AA64MMFR2, LSM, false, MRS_LOWER, 0, id_aa64mmfr2_lsm),
1487 MRS_FIELD(ID_AA64MMFR2, UAO, false, MRS_LOWER, 0, id_aa64mmfr2_uao),
1488 MRS_FIELD(ID_AA64MMFR2, CnP, false, MRS_LOWER, 0, id_aa64mmfr2_cnp),
1489 MRS_FIELD_END,
1490 };
1491
1492
1493 /* ID_AA64MMFR2_EL1 */
1494 static const struct mrs_field_value id_aa64mmfr3_spec_fpacc[] = {
1495 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, Spec_FPACC, NONE, IMPL),
1496 MRS_FIELD_VALUE_END,
1497 };
1498
1499 static const struct mrs_field_value id_aa64mmfr3_aderr[] = {
1500 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, ADERR, NONE, SOME),
1501 MRS_FIELD_VALUE_END,
1502 };
1503
1504 static const struct mrs_field_value id_aa64mmfr3_sderr[] = {
1505 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, SDERR, NONE, ALL),
1506 MRS_FIELD_VALUE_END,
1507 };
1508
1509 static const struct mrs_field_value id_aa64mmfr3_anerr[] = {
1510 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, ANERR, NONE, SOME),
1511 MRS_FIELD_VALUE_END,
1512 };
1513
1514 static const struct mrs_field_value id_aa64mmfr3_snerr[] = {
1515 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, SNERR, NONE, ALL),
1516 MRS_FIELD_VALUE_END,
1517 };
1518
1519 static const struct mrs_field_value id_aa64mmfr3_mec[] = {
1520 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, MEC, NONE, IMPL),
1521 MRS_FIELD_VALUE_END,
1522 };
1523
1524 static const struct mrs_field_value id_aa64mmfr3_aie[] = {
1525 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, AIE, NONE, IMPL),
1526 MRS_FIELD_VALUE_END,
1527 };
1528
1529 static const struct mrs_field_value id_aa64mmfr3_s2poe[] = {
1530 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, S2POE, NONE, IMPL),
1531 MRS_FIELD_VALUE_END,
1532 };
1533
1534 static const struct mrs_field_value id_aa64mmfr3_s1poe[] = {
1535 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, S1POE, NONE, IMPL),
1536 MRS_FIELD_VALUE_END,
1537 };
1538
1539 static const struct mrs_field_value id_aa64mmfr3_s2pie[] = {
1540 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, S2PIE, NONE, IMPL),
1541 MRS_FIELD_VALUE_END,
1542 };
1543
1544 static const struct mrs_field_value id_aa64mmfr3_s1pie[] = {
1545 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, S1PIE, NONE, IMPL),
1546 MRS_FIELD_VALUE_END,
1547 };
1548
1549 static const struct mrs_field_value id_aa64mmfr3_sctlrx[] = {
1550 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, SCTLRX, NONE, IMPL),
1551 MRS_FIELD_VALUE_END,
1552 };
1553
1554 static const struct mrs_field_value id_aa64mmfr3_tcrx[] = {
1555 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, TCRX, NONE, IMPL),
1556 MRS_FIELD_VALUE_END,
1557 };
1558
1559 static const struct mrs_field id_aa64mmfr3_fields[] = {
1560 MRS_FIELD(ID_AA64MMFR3, Spec_FPACC, false, MRS_LOWER, 0,
1561 id_aa64mmfr3_spec_fpacc),
1562 MRS_FIELD(ID_AA64MMFR3, ADERR, false, MRS_LOWER, 0, id_aa64mmfr3_aderr),
1563 MRS_FIELD(ID_AA64MMFR3, SDERR, false, MRS_LOWER, 0, id_aa64mmfr3_sderr),
1564 MRS_FIELD(ID_AA64MMFR3, ANERR, false, MRS_LOWER, 0, id_aa64mmfr3_anerr),
1565 MRS_FIELD(ID_AA64MMFR3, SNERR, false, MRS_LOWER, 0, id_aa64mmfr3_snerr),
1566 MRS_FIELD(ID_AA64MMFR3, MEC, false, MRS_LOWER, 0, id_aa64mmfr3_mec),
1567 MRS_FIELD(ID_AA64MMFR3, AIE, false, MRS_LOWER, 0, id_aa64mmfr3_aie),
1568 MRS_FIELD(ID_AA64MMFR3, S2POE, false, MRS_LOWER, 0, id_aa64mmfr3_s2poe),
1569 MRS_FIELD(ID_AA64MMFR3, S1POE, false, MRS_LOWER, 0, id_aa64mmfr3_s1poe),
1570 MRS_FIELD(ID_AA64MMFR3, S2PIE, false, MRS_LOWER, 0, id_aa64mmfr3_s2pie),
1571 MRS_FIELD(ID_AA64MMFR3, S1PIE, false, MRS_LOWER, 0, id_aa64mmfr3_s1pie),
1572 MRS_FIELD(ID_AA64MMFR3, SCTLRX, false, MRS_LOWER, 0,
1573 id_aa64mmfr3_sctlrx),
1574 MRS_FIELD(ID_AA64MMFR3, TCRX, false, MRS_LOWER, 0, id_aa64mmfr3_tcrx),
1575 MRS_FIELD_END,
1576 };
1577
1578
1579 /* ID_AA64MMFR4_EL1 */
1580 static const struct mrs_field id_aa64mmfr4_fields[] = {
1581 MRS_FIELD_END,
1582 };
1583
1584
1585 /* ID_AA64PFR0_EL1 */
1586 static const struct mrs_field_value id_aa64pfr0_csv3[] = {
1587 MRS_FIELD_VALUE(ID_AA64PFR0_CSV3_NONE, ""),
1588 MRS_FIELD_VALUE(ID_AA64PFR0_CSV3_ISOLATED, "CSV3"),
1589 MRS_FIELD_VALUE_END,
1590 };
1591
1592 static const struct mrs_field_value id_aa64pfr0_csv2[] = {
1593 MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_NONE, ""),
1594 MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_ISOLATED, "CSV2"),
1595 MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_SCXTNUM, "CSV2_2"),
1596 MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_3, "CSV2_3"),
1597 MRS_FIELD_VALUE_END,
1598 };
1599
1600 static const struct mrs_field_value id_aa64pfr0_rme[] = {
1601 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, RME, NONE, IMPL),
1602 MRS_FIELD_VALUE_END,
1603 };
1604
1605 static const struct mrs_field_value id_aa64pfr0_dit[] = {
1606 MRS_FIELD_VALUE(ID_AA64PFR0_DIT_NONE, ""),
1607 MRS_FIELD_VALUE(ID_AA64PFR0_DIT_PSTATE, "PSTATE.DIT"),
1608 MRS_FIELD_VALUE_END,
1609 };
1610
1611 static const struct mrs_field_hwcap id_aa64pfr0_dit_caps[] = {
1612 MRS_HWCAP(1, HWCAP_DIT, ID_AA64PFR0_DIT_PSTATE),
1613 MRS_HWCAP_END
1614 };
1615
1616 static const struct mrs_field_value id_aa64pfr0_amu[] = {
1617 MRS_FIELD_VALUE(ID_AA64PFR0_AMU_NONE, ""),
1618 MRS_FIELD_VALUE(ID_AA64PFR0_AMU_V1, "AMUv1"),
1619 MRS_FIELD_VALUE(ID_AA64PFR0_AMU_V1_1, "AMUv1p1"),
1620 MRS_FIELD_VALUE_END,
1621 };
1622
1623 static const struct mrs_field_value id_aa64pfr0_mpam[] = {
1624 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, MPAM, NONE, IMPL),
1625 MRS_FIELD_VALUE_END,
1626 };
1627
1628 static const struct mrs_field_value id_aa64pfr0_sel2[] = {
1629 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, SEL2, NONE, IMPL),
1630 MRS_FIELD_VALUE_END,
1631 };
1632
1633 static const struct mrs_field_value id_aa64pfr0_sve[] = {
1634 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, SVE, NONE, IMPL),
1635 MRS_FIELD_VALUE_END,
1636 };
1637
1638 static const struct mrs_field_hwcap id_aa64pfr0_sve_caps[] = {
1639 MRS_HWCAP(1, HWCAP_SVE, ID_AA64PFR0_SVE_IMPL),
1640 MRS_HWCAP_END
1641 };
1642
1643 static const struct mrs_field_value id_aa64pfr0_ras[] = {
1644 MRS_FIELD_VALUE(ID_AA64PFR0_RAS_NONE, ""),
1645 MRS_FIELD_VALUE(ID_AA64PFR0_RAS_IMPL, "RAS"),
1646 MRS_FIELD_VALUE(ID_AA64PFR0_RAS_8_4, "RAS v8.4"),
1647 MRS_FIELD_VALUE(ID_AA64PFR0_RAS_8_9, "RAS v8.9"),
1648 MRS_FIELD_VALUE_END,
1649 };
1650
1651 static const struct mrs_field_value id_aa64pfr0_gic[] = {
1652 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, GIC, CPUIF_NONE, CPUIF_EN),
1653 MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_NONE, ""),
1654 MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_EN, "GIC"),
1655 MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_4_1, "GIC 4.1"),
1656 MRS_FIELD_VALUE_END,
1657 };
1658
1659 static const struct mrs_field_value id_aa64pfr0_advsimd[] = {
1660 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, AdvSIMD, NONE, IMPL),
1661 MRS_FIELD_VALUE(ID_AA64PFR0_AdvSIMD_HP, "AdvSIMD+HP"),
1662 MRS_FIELD_VALUE_END,
1663 };
1664
1665 static const struct mrs_field_hwcap id_aa64pfr0_advsimd_caps[] = {
1666 MRS_HWCAP(1, HWCAP_ASIMD, ID_AA64PFR0_AdvSIMD_IMPL),
1667 MRS_HWCAP(1, HWCAP_ASIMDHP, ID_AA64PFR0_AdvSIMD_HP),
1668 MRS_HWCAP_END
1669 };
1670
1671 static const struct mrs_field_value id_aa64pfr0_fp[] = {
1672 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, FP, NONE, IMPL),
1673 MRS_FIELD_VALUE(ID_AA64PFR0_FP_HP, "FP+HP"),
1674 MRS_FIELD_VALUE_END,
1675 };
1676
1677 static const struct mrs_field_hwcap id_aa64pfr0_fp_caps[] = {
1678 MRS_HWCAP(1, HWCAP_FP, ID_AA64PFR0_FP_IMPL),
1679 MRS_HWCAP(1, HWCAP_FPHP, ID_AA64PFR0_FP_HP),
1680 MRS_HWCAP_END
1681 };
1682
1683 static const struct mrs_field_value id_aa64pfr0_el3[] = {
1684 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, EL3, NONE, 64),
1685 MRS_FIELD_VALUE(ID_AA64PFR0_EL3_64_32, "EL3 32"),
1686 MRS_FIELD_VALUE_END,
1687 };
1688
1689 static const struct mrs_field_value id_aa64pfr0_el2[] = {
1690 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, EL2, NONE, 64),
1691 MRS_FIELD_VALUE(ID_AA64PFR0_EL2_64_32, "EL2 32"),
1692 MRS_FIELD_VALUE_END,
1693 };
1694
1695 static const struct mrs_field_value id_aa64pfr0_el1[] = {
1696 MRS_FIELD_VALUE(ID_AA64PFR0_EL1_64, "EL1"),
1697 MRS_FIELD_VALUE(ID_AA64PFR0_EL1_64_32, "EL1 32"),
1698 MRS_FIELD_VALUE_END,
1699 };
1700
1701 static const struct mrs_field_value id_aa64pfr0_el0[] = {
1702 MRS_FIELD_VALUE(ID_AA64PFR0_EL0_64, "EL0"),
1703 MRS_FIELD_VALUE(ID_AA64PFR0_EL0_64_32, "EL0 32"),
1704 MRS_FIELD_VALUE_END,
1705 };
1706
1707 static const struct mrs_field id_aa64pfr0_fields[] = {
1708 MRS_FIELD(ID_AA64PFR0, CSV3, false, MRS_LOWER, 0, id_aa64pfr0_csv3),
1709 MRS_FIELD(ID_AA64PFR0, CSV2, false, MRS_LOWER, 0, id_aa64pfr0_csv2),
1710 MRS_FIELD(ID_AA64PFR0, RME, false, MRS_LOWER, 0, id_aa64pfr0_rme),
1711 MRS_FIELD_HWCAP(ID_AA64PFR0, DIT, false, MRS_LOWER, MRS_USERSPACE,
1712 id_aa64pfr0_dit, id_aa64pfr0_dit_caps),
1713 MRS_FIELD(ID_AA64PFR0, AMU, false, MRS_LOWER, 0, id_aa64pfr0_amu),
1714 MRS_FIELD(ID_AA64PFR0, MPAM, false, MRS_LOWER, 0, id_aa64pfr0_mpam),
1715 MRS_FIELD(ID_AA64PFR0, SEL2, false, MRS_LOWER, 0, id_aa64pfr0_sel2),
1716 MRS_FIELD_HWCAP(ID_AA64PFR0, SVE, false, MRS_LOWER,
1717 MRS_FREEBSD, id_aa64pfr0_sve, id_aa64pfr0_sve_caps),
1718 MRS_FIELD(ID_AA64PFR0, RAS, false, MRS_LOWER, 0, id_aa64pfr0_ras),
1719 MRS_FIELD(ID_AA64PFR0, GIC, false, MRS_LOWER, 0, id_aa64pfr0_gic),
1720 MRS_FIELD_HWCAP(ID_AA64PFR0, AdvSIMD, true, MRS_LOWER, MRS_USERSPACE,
1721 id_aa64pfr0_advsimd, id_aa64pfr0_advsimd_caps),
1722 MRS_FIELD_HWCAP(ID_AA64PFR0, FP, true, MRS_LOWER, MRS_USERSPACE,
1723 id_aa64pfr0_fp, id_aa64pfr0_fp_caps),
1724 MRS_FIELD(ID_AA64PFR0, EL3, false, MRS_LOWER, 0, id_aa64pfr0_el3),
1725 MRS_FIELD(ID_AA64PFR0, EL2, false, MRS_LOWER, 0, id_aa64pfr0_el2),
1726 MRS_FIELD(ID_AA64PFR0, EL1, false, MRS_LOWER, MRS_USERSPACE,
1727 id_aa64pfr0_el1),
1728 MRS_FIELD(ID_AA64PFR0, EL0, false, MRS_LOWER, MRS_USERSPACE,
1729 id_aa64pfr0_el0),
1730 MRS_FIELD_END,
1731 };
1732
1733
1734 /* ID_AA64PFR1_EL1 */
1735 static const struct mrs_field_value id_aa64pfr1_pfar[] = {
1736 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR1, PFAR, NONE, IMPL),
1737 MRS_FIELD_VALUE_END,
1738 };
1739
1740 static const struct mrs_field_value id_aa64pfr1_df2[] = {
1741 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR1, DF2, NONE, IMPL),
1742 MRS_FIELD_VALUE_END,
1743 };
1744
1745 static const struct mrs_field_value id_aa64pfr1_mtex[] = {
1746 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR1, MTEX, NONE, IMPL),
1747 MRS_FIELD_VALUE_END,
1748 };
1749
1750 static const struct mrs_field_value id_aa64pfr1_the[] = {
1751 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR1, THE, NONE, IMPL),
1752 MRS_FIELD_VALUE_END,
1753 };
1754
1755 static const struct mrs_field_value id_aa64pfr1_mtefrac[] = {
1756 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR1, MTE_frac, NONE, IMPL),
1757 MRS_FIELD_VALUE_END,
1758 };
1759
1760 static const struct mrs_field_value id_aa64pfr1_nmi[] = {
1761 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR1, NMI, NONE, IMPL),
1762 MRS_FIELD_VALUE_END,
1763 };
1764
1765 static const struct mrs_field_value id_aa64pfr1_csv2_frac[] = {
1766 MRS_FIELD_VALUE(ID_AA64PFR1_CSV2_frac_p0, ""),
1767 MRS_FIELD_VALUE(ID_AA64PFR1_CSV2_frac_p1, "CSV2 p1"),
1768 MRS_FIELD_VALUE(ID_AA64PFR1_CSV2_frac_p2, "CSV2 p2"),
1769 MRS_FIELD_VALUE_END,
1770 };
1771
1772 static const struct mrs_field_value id_aa64pfr1_rndr_trap[] = {
1773 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR1, RNDR_trap, NONE, IMPL),
1774 MRS_FIELD_VALUE_END,
1775 };
1776
1777 static const struct mrs_field_value id_aa64pfr1_sme[] = {
1778 MRS_FIELD_VALUE(ID_AA64PFR1_SME_NONE, ""),
1779 MRS_FIELD_VALUE(ID_AA64PFR1_SME_SME, "SME"),
1780 MRS_FIELD_VALUE(ID_AA64PFR1_SME_SME2, "SME2"),
1781 MRS_FIELD_VALUE_END,
1782 };
1783
1784 static const struct mrs_field_value id_aa64pfr1_mpam_frac[] = {
1785 MRS_FIELD_VALUE(ID_AA64PFR1_MPAM_frac_p0, ""),
1786 MRS_FIELD_VALUE(ID_AA64PFR1_MPAM_frac_p1, "MPAM p1"),
1787 MRS_FIELD_VALUE_END,
1788 };
1789
1790 static const struct mrs_field_value id_aa64pfr1_ras_frac[] = {
1791 MRS_FIELD_VALUE(ID_AA64PFR1_RAS_frac_p0, ""),
1792 MRS_FIELD_VALUE(ID_AA64PFR1_RAS_frac_p1, "RAS p1"),
1793 MRS_FIELD_VALUE_END,
1794 };
1795
1796 static const struct mrs_field_value id_aa64pfr1_mte[] = {
1797 MRS_FIELD_VALUE(ID_AA64PFR1_MTE_NONE, ""),
1798 MRS_FIELD_VALUE(ID_AA64PFR1_MTE_MTE, "MTE"),
1799 MRS_FIELD_VALUE(ID_AA64PFR1_MTE_MTE2, "MTE2"),
1800 MRS_FIELD_VALUE(ID_AA64PFR1_MTE_MTE3, "MTE3"),
1801 MRS_FIELD_VALUE_END,
1802 };
1803
1804 static const struct mrs_field_value id_aa64pfr1_ssbs[] = {
1805 MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_NONE, ""),
1806 MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_PSTATE, "PSTATE.SSBS"),
1807 MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_PSTATE_MSR, "PSTATE.SSBS MSR"),
1808 MRS_FIELD_VALUE_END,
1809 };
1810
1811 static const struct mrs_field_hwcap id_aa64pfr1_ssbs_caps[] = {
1812 MRS_HWCAP(1, HWCAP_SSBS, ID_AA64PFR1_SSBS_PSTATE),
1813 MRS_HWCAP_END
1814 };
1815
1816 static const struct mrs_field_value id_aa64pfr1_bt[] = {
1817 MRS_FIELD_VALUE(ID_AA64PFR1_BT_NONE, ""),
1818 MRS_FIELD_VALUE(ID_AA64PFR1_BT_IMPL, "BTI"),
1819 MRS_FIELD_VALUE_END,
1820 };
1821
1822 static const struct mrs_field_hwcap id_aa64pfr1_bt_caps[] = {
1823 MRS_HWCAP(2, HWCAP2_BTI, ID_AA64PFR1_BT_IMPL),
1824 MRS_HWCAP_END
1825 };
1826
1827 static const struct mrs_field id_aa64pfr1_fields[] = {
1828 MRS_FIELD(ID_AA64PFR1, PFAR, false, MRS_LOWER, 0, id_aa64pfr1_pfar),
1829 MRS_FIELD(ID_AA64PFR1, DF2, false, MRS_LOWER, 0, id_aa64pfr1_df2),
1830 MRS_FIELD(ID_AA64PFR1, MTEX, false, MRS_LOWER, 0, id_aa64pfr1_mtex),
1831 MRS_FIELD(ID_AA64PFR1, THE, false, MRS_LOWER, 0, id_aa64pfr1_the),
1832 MRS_FIELD(ID_AA64PFR1, MTE_frac, false, MRS_LOWER, 0, id_aa64pfr1_mtefrac),
1833 MRS_FIELD(ID_AA64PFR1, NMI, false, MRS_LOWER, 0, id_aa64pfr1_nmi),
1834 MRS_FIELD(ID_AA64PFR1, CSV2_frac, false, MRS_LOWER, 0,
1835 id_aa64pfr1_csv2_frac),
1836 MRS_FIELD(ID_AA64PFR1, RNDR_trap, false, MRS_LOWER, 0,
1837 id_aa64pfr1_rndr_trap),
1838 MRS_FIELD(ID_AA64PFR1, SME, false, MRS_LOWER, 0, id_aa64pfr1_sme),
1839 MRS_FIELD(ID_AA64PFR1, MPAM_frac, false, MRS_LOWER, 0,
1840 id_aa64pfr1_mpam_frac),
1841 MRS_FIELD(ID_AA64PFR1, RAS_frac, false, MRS_LOWER, 0,
1842 id_aa64pfr1_ras_frac),
1843 MRS_FIELD(ID_AA64PFR1, MTE, false, MRS_LOWER, 0, id_aa64pfr1_mte),
1844 MRS_FIELD_HWCAP(ID_AA64PFR1, SSBS, false, MRS_LOWER, MRS_USERSPACE,
1845 id_aa64pfr1_ssbs, id_aa64pfr1_ssbs_caps),
1846 MRS_FIELD_HWCAP(ID_AA64PFR1, BT, false, MRS_LOWER,
1847 MRS_FREEBSD, id_aa64pfr1_bt, id_aa64pfr1_bt_caps),
1848 MRS_FIELD_END,
1849 };
1850
1851
1852 /* ID_AA64PFR2_EL1 */
1853 static const struct mrs_field id_aa64pfr2_fields[] = {
1854 MRS_FIELD_END,
1855 };
1856
1857
1858 /* ID_AA64ZFR0_EL1 */
1859 static const struct mrs_field_value id_aa64zfr0_f64mm[] = {
1860 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, F64MM, NONE, IMPL),
1861 MRS_FIELD_VALUE_END,
1862 };
1863
1864 static const struct mrs_field_hwcap id_aa64zfr0_f64mm_caps[] = {
1865 MRS_HWCAP(2, HWCAP2_SVEF64MM, ID_AA64ZFR0_F64MM_IMPL),
1866 MRS_HWCAP_END,
1867 };
1868
1869 static const struct mrs_field_value id_aa64zfr0_f32mm[] = {
1870 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, F32MM, NONE, IMPL),
1871 MRS_FIELD_VALUE_END,
1872 };
1873
1874 static const struct mrs_field_hwcap id_aa64zfr0_f32mm_caps[] = {
1875 MRS_HWCAP(2, HWCAP2_SVEF32MM, ID_AA64ZFR0_F32MM_IMPL),
1876 MRS_HWCAP_END,
1877 };
1878
1879 static const struct mrs_field_value id_aa64zfr0_i8mm[] = {
1880 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, I8MM, NONE, IMPL),
1881 MRS_FIELD_VALUE_END,
1882 };
1883
1884 static const struct mrs_field_hwcap id_aa64zfr0_i8mm_caps[] = {
1885 MRS_HWCAP(2, HWCAP2_SVEI8MM, ID_AA64ZFR0_I8MM_IMPL),
1886 MRS_HWCAP_END,
1887 };
1888
1889 static const struct mrs_field_value id_aa64zfr0_sm4[] = {
1890 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, SM4, NONE, IMPL),
1891 MRS_FIELD_VALUE_END,
1892 };
1893
1894 static const struct mrs_field_hwcap id_aa64zfr0_sm4_caps[] = {
1895 MRS_HWCAP(2, HWCAP2_SVESM4, ID_AA64ZFR0_SM4_IMPL),
1896 MRS_HWCAP_END,
1897 };
1898
1899 static const struct mrs_field_value id_aa64zfr0_sha3[] = {
1900 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, SHA3, NONE, IMPL),
1901 MRS_FIELD_VALUE_END,
1902 };
1903
1904 static const struct mrs_field_hwcap id_aa64zfr0_sha3_caps[] = {
1905 MRS_HWCAP(2, HWCAP2_SVESHA3, ID_AA64ZFR0_SHA3_IMPL),
1906 MRS_HWCAP_END,
1907 };
1908
1909 static const struct mrs_field_value id_aa64zfr0_bf16[] = {
1910 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, BF16, NONE, BASE),
1911 MRS_FIELD_VALUE(ID_AA64ZFR0_BF16_EBF, "BF16+EBF"),
1912 MRS_FIELD_VALUE_END,
1913 };
1914
1915 static const struct mrs_field_hwcap id_aa64zfr0_bf16_caps[] = {
1916 MRS_HWCAP(2, HWCAP2_SVEBF16, ID_AA64ZFR0_BF16_BASE),
1917 MRS_HWCAP(2, HWCAP2_SVE_EBF16, ID_AA64ZFR0_BF16_EBF),
1918 MRS_HWCAP_END,
1919 };
1920
1921 static const struct mrs_field_value id_aa64zfr0_bitperm[] = {
1922 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, BitPerm, NONE, IMPL),
1923 MRS_FIELD_VALUE_END,
1924 };
1925
1926 static const struct mrs_field_hwcap id_aa64zfr0_bitperm_caps[] = {
1927 MRS_HWCAP(2, HWCAP2_SVEBITPERM, ID_AA64ZFR0_BitPerm_IMPL),
1928 MRS_HWCAP_END,
1929 };
1930
1931 static const struct mrs_field_value id_aa64zfr0_aes[] = {
1932 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, AES, NONE, BASE),
1933 MRS_FIELD_VALUE(ID_AA64ZFR0_AES_PMULL, "AES+PMULL"),
1934 MRS_FIELD_VALUE_END,
1935 };
1936
1937 static const struct mrs_field_hwcap id_aa64zfr0_aes_caps[] = {
1938 MRS_HWCAP(2, HWCAP2_SVEAES, ID_AA64ZFR0_AES_BASE),
1939 MRS_HWCAP(2, HWCAP2_SVEPMULL, ID_AA64ZFR0_AES_PMULL),
1940 MRS_HWCAP_END,
1941 };
1942
1943 static const struct mrs_field_value id_aa64zfr0_svever[] = {
1944 MRS_FIELD_VALUE(ID_AA64ZFR0_SVEver_SVE1, "SVE1"),
1945 MRS_FIELD_VALUE(ID_AA64ZFR0_SVEver_SVE2, "SVE2"),
1946 MRS_FIELD_VALUE(ID_AA64ZFR0_SVEver_SVE2P1, "SVE2P1"),
1947 MRS_FIELD_VALUE_END,
1948 };
1949
1950 static const struct mrs_field_hwcap id_aa64zfr0_svever_caps[] = {
1951 MRS_HWCAP(2, HWCAP2_SVE2, ID_AA64ZFR0_SVEver_SVE2),
1952 MRS_HWCAP(2, HWCAP2_SVE2P1, ID_AA64ZFR0_SVEver_SVE2P1),
1953 MRS_HWCAP_END,
1954 };
1955
1956 static const struct mrs_field id_aa64zfr0_fields[] = {
1957 MRS_FIELD_HWCAP(ID_AA64ZFR0, F64MM, false, MRS_LOWER, MRS_USERSPACE,
1958 id_aa64zfr0_f64mm, id_aa64zfr0_f64mm_caps),
1959 MRS_FIELD_HWCAP(ID_AA64ZFR0, F32MM, false, MRS_LOWER, MRS_USERSPACE,
1960 id_aa64zfr0_f32mm, id_aa64zfr0_f32mm_caps),
1961 MRS_FIELD_HWCAP(ID_AA64ZFR0, I8MM, false, MRS_LOWER, MRS_USERSPACE,
1962 id_aa64zfr0_i8mm, id_aa64zfr0_i8mm_caps),
1963 MRS_FIELD_HWCAP(ID_AA64ZFR0, SM4, false, MRS_LOWER, MRS_USERSPACE,
1964 id_aa64zfr0_sm4, id_aa64zfr0_sm4_caps),
1965 MRS_FIELD_HWCAP(ID_AA64ZFR0, SHA3, false, MRS_LOWER, MRS_USERSPACE,
1966 id_aa64zfr0_sha3, id_aa64zfr0_sha3_caps),
1967 MRS_FIELD_HWCAP(ID_AA64ZFR0, BF16, false, MRS_LOWER, MRS_USERSPACE,
1968 id_aa64zfr0_bf16, id_aa64zfr0_bf16_caps),
1969 MRS_FIELD_HWCAP(ID_AA64ZFR0, BitPerm, false, MRS_LOWER, MRS_USERSPACE,
1970 id_aa64zfr0_bitperm, id_aa64zfr0_bitperm_caps),
1971 MRS_FIELD_HWCAP(ID_AA64ZFR0, AES, false, MRS_LOWER, MRS_USERSPACE,
1972 id_aa64zfr0_aes, id_aa64zfr0_aes_caps),
1973 MRS_FIELD_HWCAP(ID_AA64ZFR0, SVEver, false, MRS_LOWER, MRS_USERSPACE,
1974 id_aa64zfr0_svever, id_aa64zfr0_svever_caps),
1975 MRS_FIELD_END,
1976 };
1977
1978
1979 #ifdef COMPAT_FREEBSD32
1980 /* ID_ISAR5_EL1 */
1981 static const struct mrs_field_value id_isar5_vcma[] = {
1982 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, VCMA, NONE, IMPL),
1983 MRS_FIELD_VALUE_END,
1984 };
1985
1986 static const struct mrs_field_value id_isar5_rdm[] = {
1987 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, RDM, NONE, IMPL),
1988 MRS_FIELD_VALUE_END,
1989 };
1990
1991 static const struct mrs_field_value id_isar5_crc32[] = {
1992 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, CRC32, NONE, IMPL),
1993 MRS_FIELD_VALUE_END,
1994 };
1995
1996 static const struct mrs_field_hwcap id_isar5_crc32_caps[] = {
1997 MRS_HWCAP(2, HWCAP32_2_CRC32, ID_ISAR5_CRC32_IMPL),
1998 MRS_HWCAP_END
1999 };
2000
2001 static const struct mrs_field_value id_isar5_sha2[] = {
2002 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SHA2, NONE, IMPL),
2003 MRS_FIELD_VALUE_END,
2004 };
2005
2006 static const struct mrs_field_hwcap id_isar5_sha2_caps[] = {
2007 MRS_HWCAP(2, HWCAP32_2_SHA2, ID_ISAR5_SHA2_IMPL),
2008 MRS_HWCAP_END
2009 };
2010
2011 static const struct mrs_field_value id_isar5_sha1[] = {
2012 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SHA1, NONE, IMPL),
2013 MRS_FIELD_VALUE_END,
2014 };
2015
2016 static const struct mrs_field_hwcap id_isar5_sha1_caps[] = {
2017 MRS_HWCAP(2, HWCAP32_2_SHA1, ID_ISAR5_SHA1_IMPL),
2018 MRS_HWCAP_END
2019 };
2020
2021 static const struct mrs_field_value id_isar5_aes[] = {
2022 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, AES, NONE, BASE),
2023 MRS_FIELD_VALUE(ID_ISAR5_AES_VMULL, "AES+VMULL"),
2024 MRS_FIELD_VALUE_END,
2025 };
2026
2027 static const struct mrs_field_hwcap id_isar5_aes_caps[] = {
2028 MRS_HWCAP(2, HWCAP32_2_AES, ID_ISAR5_AES_BASE),
2029 MRS_HWCAP(2, HWCAP32_2_PMULL, ID_ISAR5_AES_VMULL),
2030 MRS_HWCAP_END
2031 };
2032
2033 static const struct mrs_field_value id_isar5_sevl[] = {
2034 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SEVL, NOP, IMPL),
2035 MRS_FIELD_VALUE_END,
2036 };
2037
2038 static const struct mrs_field id_isar5_fields[] = {
2039 MRS_FIELD(ID_ISAR5, VCMA, false, MRS_LOWER,MRS_USERSPACE,
2040 id_isar5_vcma),
2041 MRS_FIELD(ID_ISAR5, RDM, false, MRS_LOWER, MRS_USERSPACE, id_isar5_rdm),
2042 MRS_FIELD_HWCAP(ID_ISAR5, CRC32, false, MRS_LOWER, MRS_USERSPACE,
2043 id_isar5_crc32, id_isar5_crc32_caps),
2044 MRS_FIELD_HWCAP(ID_ISAR5, SHA2, false, MRS_LOWER, MRS_USERSPACE,
2045 id_isar5_sha2, id_isar5_sha2_caps),
2046 MRS_FIELD_HWCAP(ID_ISAR5, SHA1, false, MRS_LOWER, MRS_USERSPACE,
2047 id_isar5_sha1, id_isar5_sha1_caps),
2048 MRS_FIELD_HWCAP(ID_ISAR5, AES, false, MRS_LOWER, MRS_USERSPACE,
2049 id_isar5_aes, id_isar5_aes_caps),
2050 MRS_FIELD(ID_ISAR5, SEVL, false, MRS_LOWER, MRS_USERSPACE,
2051 id_isar5_sevl),
2052 MRS_FIELD_END,
2053 };
2054
2055 /* MVFR0 */
2056 static const struct mrs_field_value mvfr0_fpround[] = {
2057 MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPRound, NONE, IMPL),
2058 MRS_FIELD_VALUE_END,
2059 };
2060
2061 static const struct mrs_field_value mvfr0_fpsqrt[] = {
2062 MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPSqrt, NONE, IMPL),
2063 MRS_FIELD_VALUE_END,
2064 };
2065
2066 static const struct mrs_field_value mvfr0_fpdivide[] = {
2067 MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPDivide, NONE, IMPL),
2068 MRS_FIELD_VALUE_END,
2069 };
2070
2071 static const struct mrs_field_value mvfr0_fptrap[] = {
2072 MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPTrap, NONE, IMPL),
2073 MRS_FIELD_VALUE_END,
2074 };
2075
2076 static const struct mrs_field_value mvfr0_fpdp[] = {
2077 MRS_FIELD_VALUE(MVFR0_FPDP_NONE, ""),
2078 MRS_FIELD_VALUE(MVFR0_FPDP_VFP_v2, "DP VFPv2"),
2079 MRS_FIELD_VALUE(MVFR0_FPDP_VFP_v3_v4, "DP VFPv3+v4"),
2080 MRS_FIELD_VALUE_END,
2081 };
2082
2083 static const struct mrs_field_hwcap mvfr0_fpdp_caps[] = {
2084 MRS_HWCAP(1, HWCAP32_VFP, MVFR0_FPDP_VFP_v2),
2085 MRS_HWCAP(1, HWCAP32_VFPv3, MVFR0_FPDP_VFP_v3_v4),
2086 MRS_HWCAP_END
2087 };
2088
2089 static const struct mrs_field_value mvfr0_fpsp[] = {
2090 MRS_FIELD_VALUE(MVFR0_FPSP_NONE, ""),
2091 MRS_FIELD_VALUE(MVFR0_FPSP_VFP_v2, "SP VFPv2"),
2092 MRS_FIELD_VALUE(MVFR0_FPSP_VFP_v3_v4, "SP VFPv3+v4"),
2093 MRS_FIELD_VALUE_END,
2094 };
2095
2096 static const struct mrs_field_value mvfr0_simdreg[] = {
2097 MRS_FIELD_VALUE(MVFR0_SIMDReg_NONE, ""),
2098 MRS_FIELD_VALUE(MVFR0_SIMDReg_FP, "FP 16x64"),
2099 MRS_FIELD_VALUE(MVFR0_SIMDReg_AdvSIMD, "AdvSIMD"),
2100 MRS_FIELD_VALUE_END,
2101 };
2102
2103 static const struct mrs_field mvfr0_fields[] = {
2104 MRS_FIELD(MVFR0, FPRound, false, MRS_LOWER, MRS_USERSPACE,
2105 mvfr0_fpround),
2106 MRS_FIELD(MVFR0, FPSqrt, false, MRS_LOWER, MRS_USERSPACE,
2107 mvfr0_fpsqrt),
2108 MRS_FIELD(MVFR0, FPDivide, false, MRS_LOWER, MRS_USERSPACE,
2109 mvfr0_fpdivide),
2110 MRS_FIELD(MVFR0, FPTrap, false, MRS_LOWER, MRS_USERSPACE,
2111 mvfr0_fptrap),
2112 MRS_FIELD_HWCAP(MVFR0, FPDP, false, MRS_LOWER, MRS_USERSPACE,
2113 mvfr0_fpdp, mvfr0_fpdp_caps),
2114 MRS_FIELD(MVFR0, FPSP, false, MRS_LOWER, MRS_USERSPACE, mvfr0_fpsp),
2115 MRS_FIELD(MVFR0, SIMDReg, false, MRS_LOWER, MRS_USERSPACE,
2116 mvfr0_simdreg),
2117 MRS_FIELD_END,
2118 };
2119
2120 /* MVFR1 */
2121 static const struct mrs_field_value mvfr1_simdfmac[] = {
2122 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDFMAC, NONE, IMPL),
2123 MRS_FIELD_VALUE_END,
2124 };
2125
2126 static const struct mrs_field_hwcap mvfr1_simdfmac_caps[] = {
2127 MRS_HWCAP(1, HWCAP32_VFPv4, MVFR1_SIMDFMAC_IMPL),
2128 MRS_HWCAP_END
2129 };
2130
2131 static const struct mrs_field_value mvfr1_fphp[] = {
2132 MRS_FIELD_VALUE(MVFR1_FPHP_NONE, ""),
2133 MRS_FIELD_VALUE(MVFR1_FPHP_CONV_SP, "FPHP SP Conv"),
2134 MRS_FIELD_VALUE(MVFR1_FPHP_CONV_DP, "FPHP DP Conv"),
2135 MRS_FIELD_VALUE(MVFR1_FPHP_ARITH, "FPHP Arith"),
2136 MRS_FIELD_VALUE_END,
2137 };
2138
2139 static const struct mrs_field_value mvfr1_simdhp[] = {
2140 MRS_FIELD_VALUE(MVFR1_SIMDHP_NONE, ""),
2141 MRS_FIELD_VALUE(MVFR1_SIMDHP_CONV_SP, "SIMDHP SP Conv"),
2142 MRS_FIELD_VALUE(MVFR1_SIMDHP_ARITH, "SIMDHP Arith"),
2143 MRS_FIELD_VALUE_END,
2144 };
2145
2146 static const struct mrs_field_value mvfr1_simdsp[] = {
2147 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDSP, NONE, IMPL),
2148 MRS_FIELD_VALUE_END,
2149 };
2150
2151 static const struct mrs_field_value mvfr1_simdint[] = {
2152 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDInt, NONE, IMPL),
2153 MRS_FIELD_VALUE_END,
2154 };
2155
2156 static const struct mrs_field_value mvfr1_simdls[] = {
2157 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDLS, NONE, IMPL),
2158 MRS_FIELD_VALUE_END,
2159 };
2160
2161 static const struct mrs_field_hwcap mvfr1_simdls_caps[] = {
2162 MRS_HWCAP(1, HWCAP32_VFPv4, MVFR1_SIMDFMAC_IMPL),
2163 MRS_HWCAP_END
2164 };
2165
2166 static const struct mrs_field_value mvfr1_fpdnan[] = {
2167 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, FPDNaN, NONE, IMPL),
2168 MRS_FIELD_VALUE_END,
2169 };
2170
2171 static const struct mrs_field_value mvfr1_fpftz[] = {
2172 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, FPFtZ, NONE, IMPL),
2173 MRS_FIELD_VALUE_END,
2174 };
2175
2176 static const struct mrs_field mvfr1_fields[] = {
2177 MRS_FIELD_HWCAP(MVFR1, SIMDFMAC, false, MRS_LOWER, MRS_USERSPACE,
2178 mvfr1_simdfmac, mvfr1_simdfmac_caps),
2179 MRS_FIELD(MVFR1, FPHP, false, MRS_LOWER, MRS_USERSPACE, mvfr1_fphp),
2180 MRS_FIELD(MVFR1, SIMDHP, false, MRS_LOWER, MRS_USERSPACE, mvfr1_simdhp),
2181 MRS_FIELD(MVFR1, SIMDSP, false, MRS_LOWER, MRS_USERSPACE, mvfr1_simdsp),
2182 MRS_FIELD(MVFR1, SIMDInt, false, MRS_LOWER, MRS_USERSPACE,
2183 mvfr1_simdint),
2184 MRS_FIELD_HWCAP(MVFR1, SIMDLS, false, MRS_LOWER, MRS_USERSPACE,
2185 mvfr1_simdls, mvfr1_simdls_caps),
2186 MRS_FIELD(MVFR1, FPDNaN, false, MRS_LOWER, MRS_USERSPACE,
2187 mvfr1_fpdnan),
2188 MRS_FIELD(MVFR1, FPFtZ, false, MRS_LOWER, MRS_USERSPACE,
2189 mvfr1_fpftz),
2190 MRS_FIELD_END,
2191 };
2192 #endif /* COMPAT_FREEBSD32 */
2193
2194 struct mrs_user_reg {
2195 u_int reg;
2196 u_int CRm;
2197 u_int Op2;
2198 bool is64bit;
2199 size_t offset;
2200 const struct mrs_field *fields;
2201 };
2202
2203 #define USER_REG(name, field_name, _is64bit) \
2204 { \
2205 .reg = name, \
2206 .CRm = name##_CRm, \
2207 .Op2 = name##_op2, \
2208 .offset = __offsetof(struct cpu_desc, field_name), \
2209 .fields = field_name##_fields, \
2210 .is64bit = _is64bit, \
2211 }
2212 static const struct mrs_user_reg user_regs[] = {
2213 USER_REG(ID_AA64AFR0_EL1, id_aa64afr0, true),
2214 USER_REG(ID_AA64AFR1_EL1, id_aa64afr1, true),
2215
2216 USER_REG(ID_AA64DFR0_EL1, id_aa64dfr0, true),
2217 USER_REG(ID_AA64DFR1_EL1, id_aa64dfr1, true),
2218
2219 USER_REG(ID_AA64ISAR0_EL1, id_aa64isar0, true),
2220 USER_REG(ID_AA64ISAR1_EL1, id_aa64isar1, true),
2221 USER_REG(ID_AA64ISAR2_EL1, id_aa64isar2, true),
2222
2223 USER_REG(ID_AA64MMFR0_EL1, id_aa64mmfr0, true),
2224 USER_REG(ID_AA64MMFR1_EL1, id_aa64mmfr1, true),
2225 USER_REG(ID_AA64MMFR2_EL1, id_aa64mmfr2, true),
2226 USER_REG(ID_AA64MMFR3_EL1, id_aa64mmfr3, true),
2227 USER_REG(ID_AA64MMFR4_EL1, id_aa64mmfr4, true),
2228
2229 USER_REG(ID_AA64PFR0_EL1, id_aa64pfr0, true),
2230 USER_REG(ID_AA64PFR1_EL1, id_aa64pfr1, true),
2231 USER_REG(ID_AA64PFR2_EL1, id_aa64pfr2, true),
2232
2233 USER_REG(ID_AA64ZFR0_EL1, id_aa64zfr0, true),
2234
2235 USER_REG(CTR_EL0, ctr, true),
2236
2237 #ifdef COMPAT_FREEBSD32
2238 USER_REG(ID_ISAR5_EL1, id_isar5, false),
2239
2240 USER_REG(MVFR0_EL1, mvfr0, false),
2241 USER_REG(MVFR1_EL1, mvfr1, false),
2242 #endif /* COMPAT_FREEBSD32 */
2243 };
2244
2245 #define CPU_DESC_FIELD(desc, idx) \
2246 *(uint64_t *)((char *)&(desc) + user_regs[(idx)].offset)
2247
2248 static bool
user_ctr_has_neoverse_n1_1542419(uint32_t midr,uint64_t ctr)2249 user_ctr_has_neoverse_n1_1542419(uint32_t midr, uint64_t ctr)
2250 {
2251 /* Skip non-Neoverse-N1 */
2252 if (!CPU_MATCH(CPU_IMPL_MASK | CPU_PART_MASK, CPU_IMPL_ARM,
2253 CPU_PART_NEOVERSE_N1, 0, 0))
2254 return (false);
2255
2256 switch (CPU_VAR(midr)) {
2257 default:
2258 break;
2259 case 4:
2260 /* Fixed in r4p1 */
2261 if (CPU_REV(midr) > 0)
2262 break;
2263 /* FALLTHROUGH */
2264 case 3:
2265 /* If DIC is enabled (coherent icache) then we are affected */
2266 return (CTR_DIC_VAL(ctr) != 0);
2267 }
2268
2269 return (false);
2270 }
2271
2272 static bool
user_ctr_check(const struct cpu_feat * feat __unused,u_int midr __unused)2273 user_ctr_check(const struct cpu_feat *feat __unused, u_int midr __unused)
2274 {
2275 if (emulate_ctr)
2276 return (true);
2277
2278 if (user_ctr_has_neoverse_n1_1542419(midr, READ_SPECIALREG(ctr_el0)))
2279 return (true);
2280
2281 return (false);
2282 }
2283
2284 static bool
user_ctr_has_errata(const struct cpu_feat * feat __unused,u_int midr,u_int ** errata_list,u_int * errata_count)2285 user_ctr_has_errata(const struct cpu_feat *feat __unused, u_int midr,
2286 u_int **errata_list, u_int *errata_count)
2287 {
2288 if (user_ctr_has_neoverse_n1_1542419(midr, READ_SPECIALREG(ctr_el0))) {
2289 static u_int errata_id = 1542419;
2290
2291 *errata_list = &errata_id;
2292 *errata_count = 1;
2293 return (true);
2294 }
2295
2296 return (false);
2297 }
2298
2299 static void
user_ctr_enable(const struct cpu_feat * feat __unused,cpu_feat_errata errata_status,u_int * errata_list,u_int errata_count)2300 user_ctr_enable(const struct cpu_feat *feat __unused,
2301 cpu_feat_errata errata_status, u_int *errata_list, u_int errata_count)
2302 {
2303 MPASS(emulate_ctr || errata_status != ERRATA_NONE);
2304
2305 /*
2306 * The Errata Management Firmware Interface may incorrectly mark
2307 * this as firmware mitigated. We should ignore that as there is
2308 * a kernel component to the mitigation.
2309 */
2310 if (errata_status != ERRATA_NONE && PCPU_GET(cpuid) == 0 &&
2311 cpu_feat_has_erratum(errata_list, errata_count, 1542419)) {
2312 /* Clear fields we will change */
2313 user_cpu_desc.ctr &= ~(CTR_DIC_MASK | CTR_ILINE_WIDTH);
2314
2315 /*
2316 * Set DIC to none so userspace will execute an 'ic ivau'
2317 * instruction that can be trapped by EL3.
2318 */
2319 user_cpu_desc.ctr |= CTR_DIC_NONE;
2320 /*
2321 * Set the i-cache line size to be page size to reduce the
2322 * number of times userspace needs to execute the 'ic ivau'
2323 * instruction. The ctr_el0.IminLine is log2 the number of
2324 * 4-byte words the instruction covers. As PAGE_SHIFT is log2
2325 * of the number of bytes in a page we need to subtract 2.
2326 */
2327 user_cpu_desc.ctr |= (PAGE_SHIFT - 2) << CTR_ILINE_SHIFT;
2328
2329 l_user_cpu_desc.ctr = user_cpu_desc.ctr;
2330 }
2331
2332 WRITE_SPECIALREG(sctlr_el1,
2333 READ_SPECIALREG(sctlr_el1) & ~SCTLR_UCT);
2334 isb();
2335 }
2336
2337 static struct cpu_feat user_ctr = {
2338 .feat_name = "Trap CTR_EL0",
2339 .feat_check = user_ctr_check,
2340 .feat_has_errata = user_ctr_has_errata,
2341 .feat_enable = user_ctr_enable,
2342 .feat_flags = CPU_FEAT_AFTER_DEV | CPU_FEAT_PER_CPU,
2343 };
2344 DATA_SET(cpu_feat_set, user_ctr);
2345
2346 static int
user_ctr_handler(vm_offset_t va,uint32_t insn,struct trapframe * frame,uint32_t esr)2347 user_ctr_handler(vm_offset_t va, uint32_t insn, struct trapframe *frame,
2348 uint32_t esr)
2349 {
2350 uint64_t value;
2351 int reg;
2352
2353 if ((insn & MRS_MASK) != MRS_VALUE)
2354 return (0);
2355
2356 /* Check if this is the ctr_el0 register */
2357 /* TODO: Add macros to armreg.h */
2358 if (mrs_Op0(insn) != 3 || mrs_Op1(insn) != 3 || mrs_CRn(insn) != 0 ||
2359 mrs_CRm(insn) != 0 || mrs_Op2(insn) != 1)
2360 return (0);
2361
2362 if (SV_CURPROC_ABI() == SV_ABI_FREEBSD)
2363 value = user_cpu_desc.ctr;
2364 else
2365 value = l_user_cpu_desc.ctr;
2366 /*
2367 * We will handle this instruction, move to the next so we
2368 * don't trap here again.
2369 */
2370 frame->tf_elr += INSN_SIZE;
2371
2372 reg = MRS_REGISTER(insn);
2373 /* If reg is 31 then write to xzr, i.e. do nothing */
2374 if (reg == 31)
2375 return (1);
2376
2377 if (reg < nitems(frame->tf_x))
2378 frame->tf_x[reg] = value;
2379 else if (reg == 30)
2380 frame->tf_lr = value;
2381
2382 return (1);
2383 }
2384
2385 static int
user_mrs_handler(vm_offset_t va,uint32_t insn,struct trapframe * frame,uint32_t esr)2386 user_mrs_handler(vm_offset_t va, uint32_t insn, struct trapframe *frame,
2387 uint32_t esr)
2388 {
2389 uint64_t value;
2390 int CRm, Op2, i, reg;
2391
2392 if ((insn & MRS_MASK) != MRS_VALUE)
2393 return (0);
2394
2395 /*
2396 * We only emulate Op0 == 3, Op1 == 0, CRn == 0, CRm == {0, 4-7}.
2397 * These are in the EL1 CPU identification space.
2398 * CRm == 0 holds MIDR_EL1, MPIDR_EL1, and REVID_EL1.
2399 * CRm == {4-7} holds the ID_AA64 registers.
2400 *
2401 * For full details see the ARMv8 ARM (ARM DDI 0487C.a)
2402 * Table D9-2 System instruction encodings for non-Debug System
2403 * register accesses.
2404 */
2405 if (mrs_Op0(insn) != 3 || mrs_Op1(insn) != 0 || mrs_CRn(insn) != 0)
2406 return (0);
2407
2408 CRm = mrs_CRm(insn);
2409 if (CRm > 7 || (CRm < 4 && CRm != 0))
2410 return (0);
2411
2412 Op2 = mrs_Op2(insn);
2413 value = 0;
2414
2415 for (i = 0; i < nitems(user_regs); i++) {
2416 if (user_regs[i].CRm == CRm && user_regs[i].Op2 == Op2) {
2417 if (SV_CURPROC_ABI() == SV_ABI_FREEBSD)
2418 value = CPU_DESC_FIELD(user_cpu_desc, i);
2419 else
2420 value = CPU_DESC_FIELD(l_user_cpu_desc, i);
2421 break;
2422 }
2423 }
2424
2425 if (CRm == 0) {
2426 switch (Op2) {
2427 case 0:
2428 value = READ_SPECIALREG(midr_el1);
2429 break;
2430 case 5:
2431 value = READ_SPECIALREG(mpidr_el1);
2432 break;
2433 case 6:
2434 value = READ_SPECIALREG(revidr_el1);
2435 break;
2436 default:
2437 return (0);
2438 }
2439 }
2440
2441 /*
2442 * We will handle this instruction, move to the next so we
2443 * don't trap here again.
2444 */
2445 frame->tf_elr += INSN_SIZE;
2446
2447 reg = MRS_REGISTER(insn);
2448 /* If reg is 31 then write to xzr, i.e. do nothing */
2449 if (reg == 31)
2450 return (1);
2451
2452 if (reg < nitems(frame->tf_x))
2453 frame->tf_x[reg] = value;
2454 else if (reg == 30)
2455 frame->tf_lr = value;
2456
2457 return (1);
2458 }
2459
2460 /*
2461 * Compares two field values that may be signed or unsigned.
2462 * Returns:
2463 * < 0 when a is less than b
2464 * = 0 when a equals b
2465 * > 0 when a is greater than b
2466 */
2467 static int
mrs_field_cmp(uint64_t a,uint64_t b,u_int shift,int width,bool sign)2468 mrs_field_cmp(uint64_t a, uint64_t b, u_int shift, int width, bool sign)
2469 {
2470 uint64_t mask;
2471
2472 KASSERT(width > 0 && width < 64, ("%s: Invalid width %d", __func__,
2473 width));
2474
2475 mask = (1ul << width) - 1;
2476 /* Move the field to the lower bits */
2477 a = (a >> shift) & mask;
2478 b = (b >> shift) & mask;
2479
2480 if (sign) {
2481 /*
2482 * The field is signed. Toggle the upper bit so the comparison
2483 * works on unsigned values as this makes positive numbers,
2484 * i.e. those with a 0 bit, larger than negative numbers,
2485 * i.e. those with a 1 bit, in an unsigned comparison.
2486 */
2487 a ^= 1ul << (width - 1);
2488 b ^= 1ul << (width - 1);
2489 }
2490
2491 return (a - b);
2492 }
2493
2494 bool
extract_user_id_field(u_int reg,u_int field_shift,uint8_t * val)2495 extract_user_id_field(u_int reg, u_int field_shift, uint8_t *val)
2496 {
2497 uint64_t value;
2498 int i;
2499
2500 for (i = 0; i < nitems(user_regs); i++) {
2501 if (user_regs[i].reg == reg) {
2502 value = CPU_DESC_FIELD(user_cpu_desc, i);
2503 *val = value >> field_shift;
2504 return (true);
2505 }
2506 }
2507
2508 return (false);
2509 }
2510
2511 bool
get_kernel_reg(u_int reg,uint64_t * val)2512 get_kernel_reg(u_int reg, uint64_t *val)
2513 {
2514 int i;
2515
2516 for (i = 0; i < nitems(user_regs); i++) {
2517 if (user_regs[i].reg == reg) {
2518 *val = CPU_DESC_FIELD(kern_cpu_desc, i);
2519 return (true);
2520 }
2521 }
2522
2523 return (false);
2524 }
2525
2526 /*
2527 * Fetch the specified register's value, ensuring that individual field values
2528 * do not exceed those in the mask.
2529 */
2530 bool
get_kernel_reg_masked(u_int reg,uint64_t * valp,uint64_t mask)2531 get_kernel_reg_masked(u_int reg, uint64_t *valp, uint64_t mask)
2532 {
2533 const struct mrs_field *fields;
2534 uint64_t val;
2535
2536 for (int i = 0; i < nitems(user_regs); i++) {
2537 if (user_regs[i].reg == reg) {
2538 val = CPU_DESC_FIELD(kern_cpu_desc, i);
2539 fields = user_regs[i].fields;
2540 for (int j = 0; fields[j].type != 0; j++) {
2541 mask = update_special_reg_field(mask,
2542 fields[j].type, val, fields[j].width,
2543 fields[j].shift, fields[j].sign);
2544 }
2545 *valp = mask;
2546 return (true);
2547 }
2548 }
2549
2550 return (false);
2551 }
2552
2553 static uint64_t
update_special_reg_field(uint64_t user_reg,u_int type,uint64_t value,u_int width,u_int shift,bool sign)2554 update_special_reg_field(uint64_t user_reg, u_int type, uint64_t value,
2555 u_int width, u_int shift, bool sign)
2556 {
2557 uint64_t cur, mask, new_val;
2558
2559 mask = ((1ul << width) - 1) << shift;
2560 cur = user_reg & mask;
2561 new_val = value & mask;
2562
2563 switch (type & MRS_TYPE_MASK) {
2564 case MRS_EXACT_IF_DIFFERENT:
2565 if (mrs_field_cmp(new_val, cur, shift, width, sign) == 0)
2566 break;
2567 /* FALLTHROUGH */
2568 case MRS_EXACT:
2569 cur = (uint64_t)MRS_SAFE_VAL(type) << shift;
2570 break;
2571 case MRS_LOWER:
2572 if (mrs_field_cmp(new_val, cur, shift, width, sign) < 0)
2573 cur = new_val;
2574 break;
2575 case MRS_HIGHER_OR_ZERO:
2576 if (cur == 0 || new_val == 0) {
2577 cur = 0;
2578 break;
2579 }
2580 /* FALLTHROUGH */
2581 case MRS_HIGHER:
2582 if (mrs_field_cmp(new_val, cur, shift, width, sign) > 0)
2583 cur = new_val;
2584 break;
2585 default:
2586 panic("Invalid field type: %d", type);
2587 }
2588
2589 user_reg &= ~mask;
2590 user_reg |= cur;
2591
2592 return (user_reg);
2593 }
2594
2595 void
update_special_regs(u_int cpu)2596 update_special_regs(u_int cpu)
2597 {
2598 struct cpu_desc *desc;
2599 const struct mrs_field *fields;
2600 uint64_t l_user_reg, user_reg, kern_reg, value;
2601 int i, j;
2602
2603 if (cpu == 0) {
2604 /* Create a user visible cpu description with safe values */
2605 memset(&user_cpu_desc, 0, sizeof(user_cpu_desc));
2606 /* Safe values for these registers */
2607 user_cpu_desc.id_aa64pfr0 = ID_AA64PFR0_AdvSIMD_NONE |
2608 ID_AA64PFR0_FP_NONE | ID_AA64PFR0_EL1_64 |
2609 ID_AA64PFR0_EL0_64;
2610 user_cpu_desc.id_aa64dfr0 = ID_AA64DFR0_DebugVer_8;
2611 /* Create the Linux user visible cpu description */
2612 memcpy(&l_user_cpu_desc, &user_cpu_desc, sizeof(user_cpu_desc));
2613 }
2614
2615 desc = get_cpu_desc(cpu);
2616 for (i = 0; i < nitems(user_regs); i++) {
2617 value = CPU_DESC_FIELD(*desc, i);
2618 if (cpu == 0) {
2619 kern_reg = value;
2620 user_reg = value;
2621 l_user_reg = value;
2622 } else {
2623 kern_reg = CPU_DESC_FIELD(kern_cpu_desc, i);
2624 user_reg = CPU_DESC_FIELD(user_cpu_desc, i);
2625 l_user_reg = CPU_DESC_FIELD(l_user_cpu_desc, i);
2626 }
2627
2628 fields = user_regs[i].fields;
2629 for (j = 0; fields[j].type != 0; j++) {
2630 u_int type;
2631
2632 /* Update the FreeBSD userspace ID register view */
2633 type = ((fields[j].type & MRS_FREEBSD) != 0) ?
2634 fields[j].type :
2635 (MRS_EXACT | (fields[j].type & MRS_SAFE_MASK));
2636 user_reg = update_special_reg_field(user_reg,
2637 type, value, fields[j].width, fields[j].shift,
2638 fields[j].sign);
2639
2640 /* Update the Linux userspace ID register view */
2641 type = ((fields[j].type & MRS_LINUX) != 0) ?
2642 fields[j].type :
2643 (MRS_EXACT | (fields[j].type & MRS_SAFE_MASK));
2644 l_user_reg = update_special_reg_field(l_user_reg,
2645 type, value, fields[j].width, fields[j].shift,
2646 fields[j].sign);
2647
2648 /* Update the kernel ID register view */
2649 kern_reg = update_special_reg_field(kern_reg,
2650 fields[j].type, value, fields[j].width,
2651 fields[j].shift, fields[j].sign);
2652 }
2653
2654 CPU_DESC_FIELD(kern_cpu_desc, i) = kern_reg;
2655 CPU_DESC_FIELD(user_cpu_desc, i) = user_reg;
2656 CPU_DESC_FIELD(l_user_cpu_desc, i) = l_user_reg;
2657 }
2658 }
2659
2660 void
cpu_desc_init(void)2661 cpu_desc_init(void)
2662 {
2663 if (mp_ncpus == 1)
2664 return;
2665
2666 /*
2667 * Allocate memory for the non-boot CPUs to store their registers.
2668 * As this is indexed by CPU ID we need to allocate space for CPUs
2669 * 1 to mp_maxid. Because of this mp_maxid is already the correct
2670 * number of elements.
2671 */
2672 cpu_desc = mallocarray(mp_maxid, sizeof(*cpu_desc), M_IDENTCPU,
2673 M_ZERO | M_WAITOK);
2674 }
2675
2676 /* HWCAP */
2677 bool __read_frequently lse_supported = false;
2678
2679 bool __read_frequently icache_aliasing = false;
2680 bool __read_frequently icache_vmid = false;
2681
2682 int64_t dcache_line_size; /* The minimum D cache line size */
2683 int64_t icache_line_size; /* The minimum I cache line size */
2684 int64_t idcache_line_size; /* The minimum cache line size */
2685
2686 /*
2687 * Find the values to export to userspace as AT_HWCAP and AT_HWCAP2.
2688 */
2689 static void
parse_cpu_features(bool is64bit,struct cpu_desc * cpu_desc,u_long * hwcap,u_long * hwcap2)2690 parse_cpu_features(bool is64bit, struct cpu_desc *cpu_desc, u_long *hwcap,
2691 u_long *hwcap2)
2692 {
2693 const struct mrs_field_hwcap *hwcaps;
2694 const struct mrs_field *fields;
2695 uint64_t min, reg;
2696 u_long *cur_hwcap;
2697 int i, j, k;
2698
2699 for (i = 0; i < nitems(user_regs); i++) {
2700 if (user_regs[i].is64bit != is64bit)
2701 continue;
2702
2703 reg = CPU_DESC_FIELD(*cpu_desc, i);
2704 fields = user_regs[i].fields;
2705 for (j = 0; fields[j].type != 0; j++) {
2706 hwcaps = fields[j].hwcaps;
2707 if (hwcaps == NULL)
2708 continue;
2709
2710 for (k = 0; hwcaps[k].hwcap_id != 0; k++) {
2711 KASSERT(hwcaps[k].hwcap_id == 1 ||
2712 hwcaps[k].hwcap_id == 2,
2713 ("%s: Invalid HWCAP ID %d", __func__,
2714 hwcaps[k].hwcap_id));
2715
2716 cur_hwcap = hwcaps[k].hwcap_id == 1 ?
2717 hwcap : hwcap2;
2718 min = hwcaps[k].min;
2719
2720 /*
2721 * If the field is greater than the minimum
2722 * value we can set the hwcap;
2723 */
2724 if (mrs_field_cmp(reg, min, fields[j].shift,
2725 4, fields[j].sign) >= 0) {
2726 *cur_hwcap |= hwcaps[k].hwcap_val;
2727 }
2728 }
2729 }
2730 }
2731 }
2732
2733 static void
identify_cpu_sysinit(void * dummy __unused)2734 identify_cpu_sysinit(void *dummy __unused)
2735 {
2736 struct cpu_desc *desc, *prev_desc;
2737 int cpu;
2738 bool dic, idc;
2739
2740 dic = (allow_dic != 0);
2741 idc = (allow_idc != 0);
2742
2743 prev_desc = NULL;
2744 CPU_FOREACH(cpu) {
2745 desc = get_cpu_desc(cpu);
2746 if (cpu != 0) {
2747 check_cpu_regs(cpu, desc, prev_desc);
2748 update_special_regs(cpu);
2749 }
2750
2751 if (CTR_DIC_VAL(desc->ctr) == 0)
2752 dic = false;
2753 if (CTR_IDC_VAL(desc->ctr) == 0)
2754 idc = false;
2755 prev_desc = desc;
2756 }
2757
2758 /* Find the values to export to userspace as AT_HWCAP and AT_HWCAP2 */
2759 parse_cpu_features(true, &user_cpu_desc, &elf_hwcap, &elf_hwcap2);
2760 parse_cpu_features(true, &l_user_cpu_desc, &linux_elf_hwcap,
2761 &linux_elf_hwcap2);
2762 #ifdef COMPAT_FREEBSD32
2763 parse_cpu_features(false, &user_cpu_desc, &elf32_hwcap, &elf32_hwcap2);
2764 #endif
2765
2766 /* We export the CPUID registers */
2767 elf_hwcap |= HWCAP_CPUID;
2768 linux_elf_hwcap |= HWCAP_CPUID;
2769
2770 #ifdef COMPAT_FREEBSD32
2771 /* Set the default caps and any that need to check multiple fields */
2772 elf32_hwcap |= parse_cpu_features_hwcap32();
2773 #endif
2774
2775 if (dic && idc) {
2776 arm64_icache_sync_range = &arm64_dic_idc_icache_sync_range;
2777 if (bootverbose)
2778 printf("Enabling DIC & IDC ICache sync\n");
2779 } else if (idc) {
2780 arm64_icache_sync_range = &arm64_idc_aliasing_icache_sync_range;
2781 if (bootverbose)
2782 printf("Enabling IDC ICache sync\n");
2783 }
2784
2785 if ((elf_hwcap & HWCAP_ATOMICS) != 0) {
2786 lse_supported = true;
2787 if (bootverbose)
2788 printf("Enabling LSE atomics in the kernel\n");
2789 }
2790 #ifdef LSE_ATOMICS
2791 if (!lse_supported)
2792 panic("CPU does not support LSE atomic instructions");
2793 #endif
2794
2795 install_undef_handler(true, user_ctr_handler);
2796 install_undef_handler(true, user_mrs_handler);
2797 }
2798 SYSINIT(identify_cpu, SI_SUB_CPU, SI_ORDER_MIDDLE, identify_cpu_sysinit, NULL);
2799
2800 static void
cpu_features_sysinit(void * dummy __unused)2801 cpu_features_sysinit(void *dummy __unused)
2802 {
2803 struct sbuf sb;
2804 struct cpu_desc *desc, *prev_desc;
2805 u_int cpu;
2806
2807 prev_desc = NULL;
2808 CPU_FOREACH(cpu) {
2809 desc = get_cpu_desc(cpu);
2810 print_cpu_features(cpu, desc, prev_desc);
2811 prev_desc = desc;
2812 }
2813
2814 /* Fill in cpu_model for the hw.model sysctl */
2815 sbuf_new(&sb, cpu_model, sizeof(cpu_model), SBUF_FIXEDLEN);
2816 print_cpu_midr(&sb, 0);
2817
2818 sbuf_finish(&sb);
2819 sbuf_delete(&sb);
2820
2821 free(cpu_desc, M_IDENTCPU);
2822 }
2823 /* Log features before APs are released and start printing to the dmesg. */
2824 SYSINIT(cpu_features, SI_SUB_SMP - 1, SI_ORDER_ANY, cpu_features_sysinit, NULL);
2825
2826 static void
tcr_set_e0pd1(void * arg __unused)2827 tcr_set_e0pd1(void *arg __unused)
2828 {
2829 uint64_t tcr;
2830
2831 tcr = READ_SPECIALREG(tcr_el1);
2832 tcr |= TCR_E0PD1;
2833 WRITE_SPECIALREG(tcr_el1, tcr);
2834 isb();
2835 }
2836
2837 /* Enable support for more recent architecture features */
2838 static void
cpu_feat_support(void * arg __unused)2839 cpu_feat_support(void *arg __unused)
2840 {
2841 /*
2842 * If FEAT_E0PD is supported use it to cause faults without a page
2843 * table walk if userspace tries to access kernel memory.
2844 */
2845 if (ID_AA64MMFR2_E0PD_VAL(kern_cpu_desc.id_aa64mmfr2) !=
2846 ID_AA64MMFR2_E0PD_NONE)
2847 smp_rendezvous(NULL, tcr_set_e0pd1, NULL, NULL);
2848 }
2849 SYSINIT(cpu_feat_support, SI_SUB_SMP, SI_ORDER_ANY, cpu_feat_support, NULL);
2850
2851 #ifdef COMPAT_FREEBSD32
2852 static u_long
parse_cpu_features_hwcap32(void)2853 parse_cpu_features_hwcap32(void)
2854 {
2855 u_long hwcap = HWCAP32_DEFAULT;
2856
2857 if ((MVFR1_SIMDLS_VAL(user_cpu_desc.mvfr1) >=
2858 MVFR1_SIMDLS_IMPL) &&
2859 (MVFR1_SIMDInt_VAL(user_cpu_desc.mvfr1) >=
2860 MVFR1_SIMDInt_IMPL) &&
2861 (MVFR1_SIMDSP_VAL(user_cpu_desc.mvfr1) >=
2862 MVFR1_SIMDSP_IMPL))
2863 hwcap |= HWCAP32_NEON;
2864
2865 return (hwcap);
2866 }
2867 #endif /* COMPAT_FREEBSD32 */
2868
2869 static void
print_register(struct sbuf * sb,const char * reg_name,uint64_t reg,void (* print_fields)(struct sbuf *,uint64_t,const void *),const void * arg)2870 print_register(struct sbuf *sb, const char *reg_name, uint64_t reg,
2871 void (*print_fields)(struct sbuf *, uint64_t, const void *),
2872 const void *arg)
2873 {
2874
2875 sbuf_printf(sb, "%29s = <", reg_name);
2876
2877 print_fields(sb, reg, arg);
2878
2879 sbuf_finish(sb);
2880 printf("%s>\n", sbuf_data(sb));
2881 sbuf_clear(sb);
2882 }
2883
2884 static void
print_id_fields(struct sbuf * sb,uint64_t reg,const void * arg)2885 print_id_fields(struct sbuf *sb, uint64_t reg, const void *arg)
2886 {
2887 const struct mrs_field *fields = arg;
2888 const struct mrs_field_value *fv;
2889 int field, i, j, printed;
2890
2891 #define SEP_STR ((printed++) == 0) ? "" : ","
2892 printed = 0;
2893 for (i = 0; fields[i].type != 0; i++) {
2894 fv = fields[i].values;
2895
2896 if (fv == NULL)
2897 goto next;
2898
2899 field = (reg & fields[i].mask) >> fields[i].shift;
2900 for (j = 0; fv[j].desc != NULL; j++) {
2901 if ((fv[j].value >> fields[i].shift) != field)
2902 continue;
2903
2904 if (fv[j].desc[0] != '\0')
2905 sbuf_printf(sb, "%s%s", SEP_STR, fv[j].desc);
2906 break;
2907 }
2908 if (fv[j].desc == NULL)
2909 sbuf_printf(sb, "%sUnknown %s(%x)", SEP_STR,
2910 fields[i].name, field);
2911
2912 next:
2913 reg &= ~(((1ul << fields[i].width) - 1) << fields[i].shift);
2914 }
2915
2916 if (reg != 0)
2917 sbuf_printf(sb, "%s%#lx", SEP_STR, reg);
2918 #undef SEP_STR
2919 }
2920
2921 static void
print_id_register(struct sbuf * sb,const char * reg_name,uint64_t reg,const struct mrs_field * fields)2922 print_id_register(struct sbuf *sb, const char *reg_name, uint64_t reg,
2923 const struct mrs_field *fields)
2924 {
2925
2926 print_register(sb, reg_name, reg, print_id_fields, fields);
2927 }
2928
2929 static void
print_cpu_midr(struct sbuf * sb,u_int cpu)2930 print_cpu_midr(struct sbuf *sb, u_int cpu)
2931 {
2932 const struct cpu_parts *cpu_partsp;
2933 const char *cpu_impl_name;
2934 const char *cpu_part_name;
2935 u_int midr;
2936 u_int impl_id;
2937 u_int part_id;
2938
2939 midr = pcpu_find(cpu)->pc_midr;
2940
2941 cpu_impl_name = NULL;
2942 cpu_partsp = NULL;
2943 impl_id = CPU_IMPL(midr);
2944 for (int i = 0; cpu_implementers[i].impl_name != NULL; i++) {
2945 if (impl_id == cpu_implementers[i].impl_id) {
2946 cpu_impl_name = cpu_implementers[i].impl_name;
2947 cpu_partsp = cpu_implementers[i].cpu_parts;
2948 break;
2949 }
2950 }
2951 /* Unknown implementer, so unknown part */
2952 if (cpu_impl_name == NULL) {
2953 sbuf_printf(sb, "Unknown Implementer (midr: %08x)", midr);
2954 return;
2955 }
2956
2957 KASSERT(cpu_partsp != NULL, ("%s: No parts table for implementer %s",
2958 __func__, cpu_impl_name));
2959
2960 cpu_part_name = NULL;
2961 part_id = CPU_PART(midr);
2962 for (int i = 0; cpu_partsp[i].part_name != NULL; i++) {
2963 if (part_id == cpu_partsp[i].part_id) {
2964 cpu_part_name = cpu_partsp[i].part_name;
2965 break;
2966 }
2967 }
2968 /* Known Implementer, Unknown part */
2969 if (cpu_part_name == NULL) {
2970 sbuf_printf(sb, "%s Unknown CPU r%dp%d (midr: %08x)",
2971 cpu_impl_name, CPU_VAR(midr), CPU_REV(midr), midr);
2972 return;
2973 }
2974
2975 sbuf_printf(sb, "%s %s r%dp%d", cpu_impl_name,
2976 cpu_part_name, CPU_VAR(midr), CPU_REV(midr));
2977 }
2978
2979 static void
print_cpu_cache(struct cpu_desc * desc,struct sbuf * sb,uint64_t ccs,bool icache,bool unified)2980 print_cpu_cache(struct cpu_desc *desc, struct sbuf *sb, uint64_t ccs,
2981 bool icache, bool unified)
2982 {
2983 size_t cache_size;
2984 size_t line_size;
2985
2986 /* LineSize is Log2(S) - 4. */
2987 line_size = 1 << ((ccs & CCSIDR_LineSize_MASK) + 4);
2988 /*
2989 * Calculate cache size (sets * ways * line size). There are different
2990 * formats depending on the FEAT_CCIDX bit in ID_AA64MMFR2 feature
2991 * register.
2992 */
2993 if ((desc->id_aa64mmfr2 & ID_AA64MMFR2_CCIDX_64))
2994 cache_size = (CCSIDR_NSETS_64(ccs) + 1) *
2995 (CCSIDR_ASSOC_64(ccs) + 1);
2996 else
2997 cache_size = (CCSIDR_NSETS(ccs) + 1) * (CCSIDR_ASSOC(ccs) + 1);
2998
2999 cache_size *= line_size;
3000 sbuf_printf(sb, "%zuKB (%s)", cache_size / 1024,
3001 icache ? "instruction" : unified ? "unified" : "data");
3002 }
3003
3004 static void
print_cpu_caches(struct sbuf * sb,struct cpu_desc * desc)3005 print_cpu_caches(struct sbuf *sb, struct cpu_desc *desc)
3006 {
3007 /* Print out each cache combination */
3008 uint64_t clidr;
3009 int i = 1;
3010 clidr = desc->clidr;
3011
3012 for (i = 0; (clidr & CLIDR_CTYPE_MASK) != 0; i++, clidr >>= 3) {
3013 int j = 0;
3014 int ctype_m = (clidr & CLIDR_CTYPE_MASK);
3015
3016 sbuf_printf(sb, " L%d cache: ", i + 1);
3017 if ((clidr & CLIDR_CTYPE_IO)) {
3018 print_cpu_cache(desc, sb, desc->ccsidr[i][j++], true,
3019 false);
3020 /* If there's more, add to the line. */
3021 if ((ctype_m & ~CLIDR_CTYPE_IO) != 0)
3022 sbuf_printf(sb, ", ");
3023 }
3024 if ((ctype_m & ~CLIDR_CTYPE_IO) != 0) {
3025 print_cpu_cache(desc, sb, desc->ccsidr[i][j], false,
3026 (clidr & CLIDR_CTYPE_UNIFIED));
3027 }
3028 sbuf_printf(sb, "\n");
3029
3030 }
3031 sbuf_finish(sb);
3032 printf("%s", sbuf_data(sb));
3033 }
3034
3035 static void
print_cpu_features(u_int cpu,struct cpu_desc * desc,struct cpu_desc * prev_desc)3036 print_cpu_features(u_int cpu, struct cpu_desc *desc,
3037 struct cpu_desc *prev_desc)
3038 {
3039 struct sbuf *sb;
3040
3041 sb = sbuf_new_auto();
3042 sbuf_printf(sb, "CPU%3u: ", cpu);
3043 print_cpu_midr(sb, cpu);
3044
3045 sbuf_cat(sb, " affinity:");
3046 switch(cpu_aff_levels) {
3047 default:
3048 case 4:
3049 sbuf_printf(sb, " %2d", CPU_AFF3(desc->mpidr));
3050 /* FALLTHROUGH */
3051 case 3:
3052 sbuf_printf(sb, " %2d", CPU_AFF2(desc->mpidr));
3053 /* FALLTHROUGH */
3054 case 2:
3055 sbuf_printf(sb, " %2d", CPU_AFF1(desc->mpidr));
3056 /* FALLTHROUGH */
3057 case 1:
3058 case 0: /* On UP this will be zero */
3059 sbuf_printf(sb, " %2d", CPU_AFF0(desc->mpidr));
3060 break;
3061 }
3062 sbuf_finish(sb);
3063 printf("%s\n", sbuf_data(sb));
3064 sbuf_clear(sb);
3065
3066 /*
3067 * There is a hardware errata where, if one CPU is performing a TLB
3068 * invalidation while another is performing a store-exclusive the
3069 * store-exclusive may return the wrong status. A workaround seems
3070 * to be to use an IPI to invalidate on each CPU, however given the
3071 * limited number of affected units (pass 1.1 is the evaluation
3072 * hardware revision), and the lack of information from Cavium
3073 * this has not been implemented.
3074 *
3075 * At the time of writing this the only information is from:
3076 * https://lkml.org/lkml/2016/8/4/722
3077 */
3078 /*
3079 * XXX: CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1 on its own also
3080 * triggers on pass 2.0+.
3081 */
3082 if (cpu == 0 && CPU_VAR(PCPU_GET(midr)) == 0 &&
3083 CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1)
3084 printf("WARNING: ThunderX Pass 1.1 detected.\nThis has known "
3085 "hardware bugs that may cause the incorrect operation of "
3086 "atomic operations.\n");
3087
3088 #define SHOULD_PRINT_REG(_reg) \
3089 (prev_desc == NULL || desc->_reg != prev_desc->_reg)
3090
3091 /* Cache Type Register */
3092 if (SHOULD_PRINT_REG(ctr))
3093 print_id_register(sb, "Cache Type", desc->ctr, ctr_fields);
3094
3095 /* AArch64 Instruction Set Attribute Register 0 */
3096 if (SHOULD_PRINT_REG(id_aa64isar0))
3097 print_id_register(sb, "Instruction Set Attributes 0",
3098 desc->id_aa64isar0, id_aa64isar0_fields);
3099
3100 /* AArch64 Instruction Set Attribute Register 1 */
3101 if (SHOULD_PRINT_REG(id_aa64isar1))
3102 print_id_register(sb, "Instruction Set Attributes 1",
3103 desc->id_aa64isar1, id_aa64isar1_fields);
3104
3105 /* AArch64 Instruction Set Attribute Register 2 */
3106 if (SHOULD_PRINT_REG(id_aa64isar2))
3107 print_id_register(sb, "Instruction Set Attributes 2",
3108 desc->id_aa64isar2, id_aa64isar2_fields);
3109
3110 /* AArch64 Processor Feature Register 0 */
3111 if (SHOULD_PRINT_REG(id_aa64pfr0))
3112 print_id_register(sb, "Processor Features 0",
3113 desc->id_aa64pfr0, id_aa64pfr0_fields);
3114
3115 /* AArch64 Processor Feature Register 1 */
3116 if (SHOULD_PRINT_REG(id_aa64pfr1))
3117 print_id_register(sb, "Processor Features 1",
3118 desc->id_aa64pfr1, id_aa64pfr1_fields);
3119
3120 /* AArch64 Processor Feature Register 2 */
3121 if (SHOULD_PRINT_REG(id_aa64pfr2))
3122 print_id_register(sb, "Processor Features 2",
3123 desc->id_aa64pfr2, id_aa64pfr2_fields);
3124
3125 /* AArch64 Memory Model Feature Register 0 */
3126 if (SHOULD_PRINT_REG(id_aa64mmfr0))
3127 print_id_register(sb, "Memory Model Features 0",
3128 desc->id_aa64mmfr0, id_aa64mmfr0_fields);
3129
3130 /* AArch64 Memory Model Feature Register 1 */
3131 if (SHOULD_PRINT_REG(id_aa64mmfr1))
3132 print_id_register(sb, "Memory Model Features 1",
3133 desc->id_aa64mmfr1, id_aa64mmfr1_fields);
3134
3135 /* AArch64 Memory Model Feature Register 2 */
3136 if (SHOULD_PRINT_REG(id_aa64mmfr2))
3137 print_id_register(sb, "Memory Model Features 2",
3138 desc->id_aa64mmfr2, id_aa64mmfr2_fields);
3139
3140 /* AArch64 Memory Model Feature Register 3 */
3141 if (SHOULD_PRINT_REG(id_aa64mmfr3))
3142 print_id_register(sb, "Memory Model Features 3",
3143 desc->id_aa64mmfr3, id_aa64mmfr3_fields);
3144
3145 /* AArch64 Memory Model Feature Register 4 */
3146 if (SHOULD_PRINT_REG(id_aa64mmfr4))
3147 print_id_register(sb, "Memory Model Features 4",
3148 desc->id_aa64mmfr4, id_aa64mmfr4_fields);
3149
3150 /* AArch64 Debug Feature Register 0 */
3151 if (SHOULD_PRINT_REG(id_aa64dfr0))
3152 print_id_register(sb, "Debug Features 0",
3153 desc->id_aa64dfr0, id_aa64dfr0_fields);
3154
3155 /* AArch64 Memory Model Feature Register 1 */
3156 if (SHOULD_PRINT_REG(id_aa64dfr1))
3157 print_id_register(sb, "Debug Features 1",
3158 desc->id_aa64dfr1, id_aa64dfr1_fields);
3159
3160 /* AArch64 Auxiliary Feature Register 0 */
3161 if (SHOULD_PRINT_REG(id_aa64afr0))
3162 print_id_register(sb, "Auxiliary Features 0",
3163 desc->id_aa64afr0, id_aa64afr0_fields);
3164
3165 /* AArch64 Auxiliary Feature Register 1 */
3166 if (SHOULD_PRINT_REG(id_aa64afr1))
3167 print_id_register(sb, "Auxiliary Features 1",
3168 desc->id_aa64afr1, id_aa64afr1_fields);
3169
3170 /* AArch64 SVE Feature Register 0 */
3171 if (desc->have_sve) {
3172 if (SHOULD_PRINT_REG(id_aa64zfr0) ||
3173 !prev_desc->have_sve) {
3174 print_id_register(sb, "SVE Features 0",
3175 desc->id_aa64zfr0, id_aa64zfr0_fields);
3176 }
3177 }
3178
3179 #ifdef COMPAT_FREEBSD32
3180 /* AArch32 Instruction Set Attribute Register 5 */
3181 if (SHOULD_PRINT_REG(id_isar5))
3182 print_id_register(sb, "AArch32 Instruction Set Attributes 5",
3183 desc->id_isar5, id_isar5_fields);
3184
3185 /* AArch32 Media and VFP Feature Register 0 */
3186 if (SHOULD_PRINT_REG(mvfr0))
3187 print_id_register(sb, "AArch32 Media and VFP Features 0",
3188 desc->mvfr0, mvfr0_fields);
3189
3190 /* AArch32 Media and VFP Feature Register 1 */
3191 if (SHOULD_PRINT_REG(mvfr1))
3192 print_id_register(sb, "AArch32 Media and VFP Features 1",
3193 desc->mvfr1, mvfr1_fields);
3194 #endif
3195 if (bootverbose)
3196 print_cpu_caches(sb, desc);
3197
3198 sbuf_delete(sb);
3199 sb = NULL;
3200 #undef SHOULD_PRINT_REG
3201 #undef SEP_STR
3202 }
3203
3204 void
identify_cache(uint64_t ctr)3205 identify_cache(uint64_t ctr)
3206 {
3207
3208 /* Identify the L1 cache type */
3209 switch (CTR_L1IP_VAL(ctr)) {
3210 case CTR_L1IP_PIPT:
3211 break;
3212 default:
3213 case CTR_L1IP_VIPT:
3214 icache_aliasing = true;
3215 break;
3216 }
3217
3218 if (dcache_line_size == 0) {
3219 KASSERT(icache_line_size == 0, ("%s: i-cacheline size set: %ld",
3220 __func__, icache_line_size));
3221
3222 /* Get the D cache line size */
3223 dcache_line_size = CTR_DLINE_SIZE(ctr);
3224 /* And the same for the I cache */
3225 icache_line_size = CTR_ILINE_SIZE(ctr);
3226
3227 idcache_line_size = MIN(dcache_line_size, icache_line_size);
3228 }
3229
3230 if (dcache_line_size != CTR_DLINE_SIZE(ctr)) {
3231 printf("WARNING: D-cacheline size mismatch %ld != %d\n",
3232 dcache_line_size, CTR_DLINE_SIZE(ctr));
3233 }
3234
3235 if (icache_line_size != CTR_ILINE_SIZE(ctr)) {
3236 printf("WARNING: I-cacheline size mismatch %ld != %d\n",
3237 icache_line_size, CTR_ILINE_SIZE(ctr));
3238 }
3239 }
3240
3241 void
identify_cpu(u_int cpu)3242 identify_cpu(u_int cpu)
3243 {
3244 struct cpu_desc *desc;
3245 uint64_t clidr;
3246
3247 desc = get_cpu_desc(cpu);
3248 /* Save affinity for current CPU */
3249 desc->mpidr = get_mpidr();
3250 CPU_AFFINITY(cpu) = desc->mpidr & CPU_AFF_MASK;
3251
3252 desc->ctr = READ_SPECIALREG(ctr_el0);
3253 desc->id_aa64dfr0 = READ_SPECIALREG(ID_AA64DFR0_EL1_REG);
3254 desc->id_aa64dfr1 = READ_SPECIALREG(ID_AA64DFR1_EL1_REG);
3255 desc->id_aa64isar0 = READ_SPECIALREG(ID_AA64ISAR0_EL1_REG);
3256 desc->id_aa64isar1 = READ_SPECIALREG(ID_AA64ISAR1_EL1_REG);
3257 desc->id_aa64isar2 = READ_SPECIALREG(ID_AA64ISAR2_EL1_REG);
3258 desc->id_aa64mmfr0 = READ_SPECIALREG(ID_AA64MMFR0_EL1_REG);
3259 desc->id_aa64mmfr1 = READ_SPECIALREG(ID_AA64MMFR1_EL1_REG);
3260 desc->id_aa64mmfr2 = READ_SPECIALREG(ID_AA64MMFR2_EL1_REG);
3261 desc->id_aa64mmfr3 = READ_SPECIALREG(ID_AA64MMFR3_EL1_REG);
3262 desc->id_aa64mmfr4 = READ_SPECIALREG(ID_AA64MMFR4_EL1_REG);
3263 desc->id_aa64pfr0 = READ_SPECIALREG(ID_AA64PFR0_EL1_REG);
3264 desc->id_aa64pfr1 = READ_SPECIALREG(ID_AA64PFR1_EL1_REG);
3265 desc->id_aa64pfr2 = READ_SPECIALREG(ID_AA64PFR2_EL1_REG);
3266
3267 /*
3268 * ID_AA64ZFR0_EL1 is only valid when at least one of:
3269 * - ID_AA64PFR0_EL1.SVE is non-zero
3270 * - ID_AA64PFR1_EL1.SME is non-zero
3271 * In other cases it is zero, but still safe to read
3272 */
3273 desc->have_sve =
3274 (ID_AA64PFR0_SVE_VAL(desc->id_aa64pfr0) != 0);
3275 desc->id_aa64zfr0 = READ_SPECIALREG(ID_AA64ZFR0_EL1_REG);
3276
3277 desc->clidr = READ_SPECIALREG(clidr_el1);
3278
3279 clidr = desc->clidr;
3280
3281 for (int i = 0; (clidr & CLIDR_CTYPE_MASK) != 0; i++, clidr >>= 3) {
3282 int j = 0;
3283 if ((clidr & CLIDR_CTYPE_IO)) {
3284 WRITE_SPECIALREG(csselr_el1,
3285 CSSELR_Level(i) | CSSELR_InD);
3286 desc->ccsidr[i][j++] =
3287 READ_SPECIALREG(ccsidr_el1);
3288 }
3289 if ((clidr & ~CLIDR_CTYPE_IO) == 0)
3290 continue;
3291 WRITE_SPECIALREG(csselr_el1, CSSELR_Level(i));
3292 desc->ccsidr[i][j] = READ_SPECIALREG(ccsidr_el1);
3293 }
3294
3295 #ifdef COMPAT_FREEBSD32
3296 /* Only read aarch32 SRs if EL0-32 is available */
3297 if (ID_AA64PFR0_EL0_VAL(desc->id_aa64pfr0) == ID_AA64PFR0_EL0_64_32) {
3298 desc->id_isar5 = READ_SPECIALREG(id_isar5_el1);
3299 desc->mvfr0 = READ_SPECIALREG(mvfr0_el1);
3300 desc->mvfr1 = READ_SPECIALREG(mvfr1_el1);
3301 }
3302 #endif
3303 }
3304
3305 static void
check_cpu_regs(u_int cpu,struct cpu_desc * desc,struct cpu_desc * prev_desc)3306 check_cpu_regs(u_int cpu, struct cpu_desc *desc, struct cpu_desc *prev_desc)
3307 {
3308 switch (cpu_aff_levels) {
3309 case 0:
3310 if (CPU_AFF0(desc->mpidr) != CPU_AFF0(prev_desc->mpidr))
3311 cpu_aff_levels = 1;
3312 /* FALLTHROUGH */
3313 case 1:
3314 if (CPU_AFF1(desc->mpidr) != CPU_AFF1(prev_desc->mpidr))
3315 cpu_aff_levels = 2;
3316 /* FALLTHROUGH */
3317 case 2:
3318 if (CPU_AFF2(desc->mpidr) != CPU_AFF2(prev_desc->mpidr))
3319 cpu_aff_levels = 3;
3320 /* FALLTHROUGH */
3321 case 3:
3322 if (CPU_AFF3(desc->mpidr) != CPU_AFF3(prev_desc->mpidr))
3323 cpu_aff_levels = 4;
3324 break;
3325 }
3326
3327 if (desc->ctr != prev_desc->ctr) {
3328 /*
3329 * If the cache is different on different cores we should
3330 * emulate for userspace to provide a uniform value
3331 */
3332 emulate_ctr = true;
3333
3334 /*
3335 * If the cache type register is different we may
3336 * have a different l1 cache type.
3337 */
3338 identify_cache(desc->ctr);
3339 }
3340 }
3341