1 /*-
2 * Copyright (c) 2014 Andrew Turner
3 * Copyright (c) 2014 The FreeBSD Foundation
4 * All rights reserved.
5 *
6 * Portions of this software were developed by Semihalf
7 * under sponsorship of the FreeBSD Foundation.
8 *
9 * Redistribution and use in source and binary forms, with or without
10 * modification, are permitted provided that the following conditions
11 * are met:
12 * 1. Redistributions of source code must retain the above copyright
13 * notice, this list of conditions and the following disclaimer.
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
28 * SUCH DAMAGE.
29 *
30 */
31
32 #include <sys/param.h>
33 #include <sys/kernel.h>
34 #include <sys/malloc.h>
35 #include <sys/proc.h>
36 #include <sys/pcpu.h>
37 #include <sys/sbuf.h>
38 #include <sys/smp.h>
39 #include <sys/sysctl.h>
40 #include <sys/sysent.h>
41 #include <sys/systm.h>
42
43 #include <machine/atomic.h>
44 #include <machine/cpu.h>
45 #include <machine/cpu_feat.h>
46 #include <machine/cpufunc.h>
47 #include <machine/elf.h>
48 #include <machine/md_var.h>
49 #include <machine/undefined.h>
50
51 static MALLOC_DEFINE(M_IDENTCPU, "CPU ID", "arm64 CPU identification memory");
52
53 struct cpu_desc;
54 #ifdef INVARIANTS
55 static bool hwcaps_set = false;
56 #endif
57
58 static void print_cpu_midr(struct sbuf *sb, u_int cpu);
59 static void print_cpu_features(u_int cpu, struct cpu_desc *desc,
60 struct cpu_desc *prev_desc);
61 static void print_cpu_caches(struct sbuf *sb, struct cpu_desc *desc);
62 #ifdef COMPAT_FREEBSD32
63 static u_long parse_cpu_features_hwcap32(void);
64 #endif
65
66 const char machine[] = "arm64";
67
68 #ifdef SCTL_MASK32
69 extern int adaptive_machine_arch;
70 #endif
71
72 static SYSCTL_NODE(_machdep, OID_AUTO, cache, CTLFLAG_RD | CTLFLAG_MPSAFE, 0,
73 "Cache management tuning");
74
75 static int allow_dic = 1;
76 SYSCTL_INT(_machdep_cache, OID_AUTO, allow_dic, CTLFLAG_RDTUN, &allow_dic, 0,
77 "Allow optimizations based on the DIC cache bit");
78
79 static int allow_idc = 1;
80 SYSCTL_INT(_machdep_cache, OID_AUTO, allow_idc, CTLFLAG_RDTUN, &allow_idc, 0,
81 "Allow optimizations based on the IDC cache bit");
82
83 static bool emulate_ctr = false;
84
85 static void check_cpu_regs(u_int cpu, struct cpu_desc *desc,
86 struct cpu_desc *prev_desc);
87 static uint64_t update_special_reg_field(uint64_t user_reg, u_int type,
88 uint64_t value, u_int width, u_int shift, bool sign);
89
90 /*
91 * The default implementation of I-cache sync assumes we have an
92 * aliasing cache until we know otherwise.
93 */
94 void (*arm64_icache_sync_range)(void *, vm_size_t) =
95 &arm64_aliasing_icache_sync_range;
96
97 static int
sysctl_hw_machine(SYSCTL_HANDLER_ARGS)98 sysctl_hw_machine(SYSCTL_HANDLER_ARGS)
99 {
100 #ifdef SCTL_MASK32
101 static const char machine32[] = "arm";
102 #endif
103 int error;
104
105 #ifdef SCTL_MASK32
106 if ((req->flags & SCTL_MASK32) != 0 && adaptive_machine_arch)
107 error = SYSCTL_OUT(req, machine32, sizeof(machine32));
108 else
109 #endif
110 error = SYSCTL_OUT(req, machine, sizeof(machine));
111 return (error);
112 }
113
114 SYSCTL_PROC(_hw, HW_MACHINE, machine, CTLTYPE_STRING | CTLFLAG_RD |
115 CTLFLAG_CAPRD | CTLFLAG_MPSAFE, NULL, 0, sysctl_hw_machine, "A",
116 "Machine class");
117
118 static char cpu_model[64];
119 SYSCTL_STRING(_hw, HW_MODEL, model, CTLFLAG_RD | CTLFLAG_CAPRD,
120 cpu_model, sizeof(cpu_model), "Machine model");
121
122 #define MAX_CACHES 8 /* Maximum number of caches supported
123 architecturally. */
124 /*
125 * Per-CPU affinity as provided in MPIDR_EL1
126 * Indexed by CPU number in logical order selected by the system.
127 * Relevant fields can be extracted using CPU_AFFn macros,
128 * Aff3.Aff2.Aff1.Aff0 construct a unique CPU address in the system.
129 *
130 * Fields used by us:
131 * Aff1 - Cluster number
132 * Aff0 - CPU number in Aff1 cluster
133 */
134 uint64_t __cpu_affinity[MAXCPU];
135 static u_int cpu_aff_levels;
136
137 struct cpu_desc {
138 uint64_t mpidr;
139 uint64_t id_aa64afr0;
140 uint64_t id_aa64afr1;
141 uint64_t id_aa64dfr0;
142 uint64_t id_aa64dfr1;
143 uint64_t id_aa64isar0;
144 uint64_t id_aa64isar1;
145 uint64_t id_aa64isar2;
146 uint64_t id_aa64mmfr0;
147 uint64_t id_aa64mmfr1;
148 uint64_t id_aa64mmfr2;
149 uint64_t id_aa64mmfr3;
150 uint64_t id_aa64mmfr4;
151 uint64_t id_aa64pfr0;
152 uint64_t id_aa64pfr1;
153 uint64_t id_aa64pfr2;
154 uint64_t id_aa64zfr0;
155 uint64_t ctr;
156 #ifdef COMPAT_FREEBSD32
157 uint64_t id_isar5;
158 uint64_t mvfr0;
159 uint64_t mvfr1;
160 #endif
161 uint64_t clidr;
162 uint32_t ccsidr[MAX_CACHES][2]; /* 2 possible types. */
163 bool have_sve;
164 };
165
166 static struct cpu_desc cpu_desc0;
167 static struct cpu_desc *cpu_desc;
168 static struct cpu_desc kern_cpu_desc;
169 static struct cpu_desc user_cpu_desc;
170 static struct cpu_desc l_user_cpu_desc;
171
172 static struct cpu_desc *
get_cpu_desc(u_int cpu)173 get_cpu_desc(u_int cpu)
174 {
175 /* The cpu_desc for CPU 0 is used before the allocator is ready. */
176 if (cpu == 0)
177 return (&cpu_desc0);
178
179 MPASS(cpu_desc != NULL);
180 return (&cpu_desc[cpu - 1]);
181 }
182
183 struct cpu_parts {
184 u_int part_id;
185 const char *part_name;
186 };
187 #define CPU_PART_NONE { 0, NULL }
188
189 struct cpu_implementers {
190 u_int impl_id;
191 const char *impl_name;
192 /*
193 * Part number is implementation defined
194 * so each vendor will have its own set of values and names.
195 */
196 const struct cpu_parts *cpu_parts;
197 };
198 #define CPU_IMPLEMENTER_NONE { 0, NULL, NULL }
199
200 /*
201 * Per-implementer table of (PartNum, CPU Name) pairs.
202 */
203 /* ARM Ltd. */
204 static const struct cpu_parts cpu_parts_arm[] = {
205 { CPU_PART_AEM_V8, "AEMv8" },
206 { CPU_PART_FOUNDATION, "Foundation-Model" },
207 { CPU_PART_CORTEX_A34, "Cortex-A34" },
208 { CPU_PART_CORTEX_A35, "Cortex-A35" },
209 { CPU_PART_CORTEX_A53, "Cortex-A53" },
210 { CPU_PART_CORTEX_A55, "Cortex-A55" },
211 { CPU_PART_CORTEX_A57, "Cortex-A57" },
212 { CPU_PART_CORTEX_A65, "Cortex-A65" },
213 { CPU_PART_CORTEX_A65AE, "Cortex-A65AE" },
214 { CPU_PART_CORTEX_A72, "Cortex-A72" },
215 { CPU_PART_CORTEX_A73, "Cortex-A73" },
216 { CPU_PART_CORTEX_A75, "Cortex-A75" },
217 { CPU_PART_CORTEX_A76, "Cortex-A76" },
218 { CPU_PART_CORTEX_A76AE, "Cortex-A76AE" },
219 { CPU_PART_CORTEX_A77, "Cortex-A77" },
220 { CPU_PART_CORTEX_A78, "Cortex-A78" },
221 { CPU_PART_CORTEX_A78AE, "Cortex-A78AE" },
222 { CPU_PART_CORTEX_A78C, "Cortex-A78C" },
223 { CPU_PART_CORTEX_A510, "Cortex-A510" },
224 { CPU_PART_CORTEX_A520, "Cortex-A520" },
225 { CPU_PART_CORTEX_A710, "Cortex-A710" },
226 { CPU_PART_CORTEX_A715, "Cortex-A715" },
227 { CPU_PART_CORTEX_A720, "Cortex-A720" },
228 { CPU_PART_CORTEX_A725, "Cortex-A725" },
229 { CPU_PART_CORTEX_X925, "Cortex-A925" },
230 { CPU_PART_CORTEX_X1, "Cortex-X1" },
231 { CPU_PART_CORTEX_X1C, "Cortex-X1C" },
232 { CPU_PART_CORTEX_X2, "Cortex-X2" },
233 { CPU_PART_CORTEX_X3, "Cortex-X3" },
234 { CPU_PART_CORTEX_X4, "Cortex-X4" },
235 { CPU_PART_C1_NANO, "C1-Nano" },
236 { CPU_PART_C1_PRO, "C1-Pro" },
237 { CPU_PART_C1_PREMIUM, "C1-Premium" },
238 { CPU_PART_C1_ULTRA, "C1-Ultra" },
239 { CPU_PART_NEOVERSE_E1, "Neoverse-E1" },
240 { CPU_PART_NEOVERSE_N1, "Neoverse-N1" },
241 { CPU_PART_NEOVERSE_N2, "Neoverse-N2" },
242 { CPU_PART_NEOVERSE_N3, "Neoverse-N3" },
243 { CPU_PART_NEOVERSE_V1, "Neoverse-V1" },
244 { CPU_PART_NEOVERSE_V2, "Neoverse-V2" },
245 { CPU_PART_NEOVERSE_V3, "Neoverse-V3" },
246 { CPU_PART_NEOVERSE_V3AE, "Neoverse-V3AE" },
247 CPU_PART_NONE,
248 };
249
250 /* Cavium */
251 static const struct cpu_parts cpu_parts_cavium[] = {
252 { CPU_PART_THUNDERX, "ThunderX" },
253 { CPU_PART_THUNDERX2, "ThunderX2" },
254 CPU_PART_NONE,
255 };
256
257 /* APM (now Ampere) */
258 static const struct cpu_parts cpu_parts_apm[] = {
259 { CPU_PART_EMAG8180, "eMAG 8180" },
260 CPU_PART_NONE,
261 };
262
263 /* Ampere */
264 static const struct cpu_parts cpu_parts_ampere[] = {
265 { CPU_PART_AMPERE1, "AmpereOne AC03" },
266 { CPU_PART_AMPERE1A, "AmpereOne AC04" },
267 CPU_PART_NONE,
268 };
269
270 /* Microsoft */
271 static const struct cpu_parts cpu_parts_microsoft[] = {
272 { CPU_PART_AZURE_COBALT_100, "Azure Cobalt 100" },
273 CPU_PART_NONE,
274 };
275
276 /* Qualcomm */
277 static const struct cpu_parts cpu_parts_qcom[] = {
278 { CPU_PART_KRYO400_GOLD, "Kryo 400 Gold" },
279 { CPU_PART_KRYO400_SILVER, "Kryo 400 Silver" },
280 CPU_PART_NONE,
281 };
282
283 /* Apple */
284 static const struct cpu_parts cpu_parts_apple[] = {
285 { CPU_PART_M1_ICESTORM, "M1 Icestorm" },
286 { CPU_PART_M1_FIRESTORM, "M1 Firestorm" },
287 { CPU_PART_M1_ICESTORM_PRO, "M1 Pro Icestorm" },
288 { CPU_PART_M1_FIRESTORM_PRO, "M1 Pro Firestorm" },
289 { CPU_PART_M1_ICESTORM_MAX, "M1 Max Icestorm" },
290 { CPU_PART_M1_FIRESTORM_MAX, "M1 Max Firestorm" },
291 { CPU_PART_M2_BLIZZARD, "M2 Blizzard" },
292 { CPU_PART_M2_AVALANCHE, "M2 Avalanche" },
293 { CPU_PART_M2_BLIZZARD_PRO, "M2 Pro Blizzard" },
294 { CPU_PART_M2_AVALANCHE_PRO, "M2 Pro Avalanche" },
295 { CPU_PART_M2_BLIZZARD_MAX, "M2 Max Blizzard" },
296 { CPU_PART_M2_AVALANCHE_MAX, "M2 Max Avalanche" },
297 CPU_PART_NONE,
298 };
299
300 /* Unknown */
301 static const struct cpu_parts cpu_parts_none[] = {
302 CPU_PART_NONE,
303 };
304
305 /*
306 * Implementers table.
307 */
308 const struct cpu_implementers cpu_implementers[] = {
309 { CPU_IMPL_AMPERE, "Ampere", cpu_parts_ampere },
310 { CPU_IMPL_APPLE, "Apple", cpu_parts_apple },
311 { CPU_IMPL_APM, "APM", cpu_parts_apm },
312 { CPU_IMPL_ARM, "ARM", cpu_parts_arm },
313 { CPU_IMPL_BROADCOM, "Broadcom", cpu_parts_none },
314 { CPU_IMPL_CAVIUM, "Cavium", cpu_parts_cavium },
315 { CPU_IMPL_DEC, "DEC", cpu_parts_none },
316 { CPU_IMPL_FREESCALE, "Freescale", cpu_parts_none },
317 { CPU_IMPL_FUJITSU, "Fujitsu", cpu_parts_none },
318 { CPU_IMPL_HISILICON, "HiSilicon", cpu_parts_none },
319 { CPU_IMPL_INFINEON, "IFX", cpu_parts_none },
320 { CPU_IMPL_INTEL, "Intel", cpu_parts_none },
321 { CPU_IMPL_MARVELL, "Marvell", cpu_parts_none },
322 { CPU_IMPL_MICROSOFT, "Microsoft", cpu_parts_microsoft },
323 { CPU_IMPL_NVIDIA, "NVIDIA", cpu_parts_none },
324 { CPU_IMPL_QUALCOMM, "Qualcomm", cpu_parts_qcom },
325 CPU_IMPLEMENTER_NONE,
326 };
327
328 #define MRS_TYPE_MASK 0xf
329 #define MRS_INVALID 0
330 #define MRS_EXACT 1
331 #define MRS_EXACT_IF_DIFFERENT 2
332 #define MRS_LOWER 3
333 #define MRS_HIGHER_OR_ZERO 4
334 #define MRS_HIGHER 5
335 #define MRS_SAFE_SHIFT 4
336 #define MRS_SAFE_MASK (0xfu << MRS_SAFE_SHIFT)
337 #define MRS_SAFE(x) (((x) << MRS_SAFE_SHIFT) & MRS_SAFE_MASK)
338 #define MRS_SAFE_VAL(x) (((x) & MRS_SAFE_MASK) >> MRS_SAFE_SHIFT)
339 #define MRS_FREEBSD (1u << 8)
340 #define MRS_LINUX (1u << 9)
341 #define MRS_USERSPACE (MRS_FREEBSD | MRS_LINUX)
342
343 struct mrs_field_value {
344 uint64_t value;
345 const char *desc;
346 };
347
348 #define MRS_FIELD_VALUE(_value, _desc) \
349 { \
350 .value = (_value), \
351 .desc = (_desc), \
352 }
353
354 #define MRS_FIELD_VALUE_NONE_IMPL(_reg, _field, _none, _impl) \
355 MRS_FIELD_VALUE(_reg ## _ ## _field ## _ ## _none, ""), \
356 MRS_FIELD_VALUE(_reg ## _ ## _field ## _ ## _impl, #_field)
357
358 #define MRS_FIELD_VALUE_COUNT(_reg, _field, _desc) \
359 MRS_FIELD_VALUE(0ul << _reg ## _ ## _field ## _SHIFT, "1 " _desc), \
360 MRS_FIELD_VALUE(1ul << _reg ## _ ## _field ## _SHIFT, "2 " _desc "s"), \
361 MRS_FIELD_VALUE(2ul << _reg ## _ ## _field ## _SHIFT, "3 " _desc "s"), \
362 MRS_FIELD_VALUE(3ul << _reg ## _ ## _field ## _SHIFT, "4 " _desc "s"), \
363 MRS_FIELD_VALUE(4ul << _reg ## _ ## _field ## _SHIFT, "5 " _desc "s"), \
364 MRS_FIELD_VALUE(5ul << _reg ## _ ## _field ## _SHIFT, "6 " _desc "s"), \
365 MRS_FIELD_VALUE(6ul << _reg ## _ ## _field ## _SHIFT, "7 " _desc "s"), \
366 MRS_FIELD_VALUE(7ul << _reg ## _ ## _field ## _SHIFT, "8 " _desc "s"), \
367 MRS_FIELD_VALUE(8ul << _reg ## _ ## _field ## _SHIFT, "9 " _desc "s"), \
368 MRS_FIELD_VALUE(9ul << _reg ## _ ## _field ## _SHIFT, "10 "_desc "s"), \
369 MRS_FIELD_VALUE(10ul<< _reg ## _ ## _field ## _SHIFT, "11 "_desc "s"), \
370 MRS_FIELD_VALUE(11ul<< _reg ## _ ## _field ## _SHIFT, "12 "_desc "s"), \
371 MRS_FIELD_VALUE(12ul<< _reg ## _ ## _field ## _SHIFT, "13 "_desc "s"), \
372 MRS_FIELD_VALUE(13ul<< _reg ## _ ## _field ## _SHIFT, "14 "_desc "s"), \
373 MRS_FIELD_VALUE(14ul<< _reg ## _ ## _field ## _SHIFT, "15 "_desc "s"), \
374 MRS_FIELD_VALUE(15ul<< _reg ## _ ## _field ## _SHIFT, "16 "_desc "s")
375
376 /*
377 * Used for printing I/D cache line sizes & CWG/ERG, as 0 is a special case
378 * in some cases the decoded string needs to be passed in.
379 */
380 #define MRS_FIELD_VALUE_CACHE(_reg, _field, _0desc, _desc) \
381 MRS_FIELD_VALUE(0ul << _reg ## _ ## _field ## _SHIFT, _0desc), \
382 MRS_FIELD_VALUE(1ul << _reg ## _ ## _field ## _SHIFT, "8 " _desc), \
383 MRS_FIELD_VALUE(2ul << _reg ## _ ## _field ## _SHIFT, "16 " _desc), \
384 MRS_FIELD_VALUE(3ul << _reg ## _ ## _field ## _SHIFT, "32 " _desc), \
385 MRS_FIELD_VALUE(4ul << _reg ## _ ## _field ## _SHIFT, "64 " _desc), \
386 MRS_FIELD_VALUE(5ul << _reg ## _ ## _field ## _SHIFT, "128 " _desc), \
387 MRS_FIELD_VALUE(6ul << _reg ## _ ## _field ## _SHIFT, "256 " _desc), \
388 MRS_FIELD_VALUE(7ul << _reg ## _ ## _field ## _SHIFT, "512 " _desc), \
389 MRS_FIELD_VALUE(8ul << _reg ## _ ## _field ## _SHIFT, "1k " _desc), \
390 MRS_FIELD_VALUE(9ul << _reg ## _ ## _field ## _SHIFT, "2k " _desc), \
391 MRS_FIELD_VALUE(10ul<< _reg ## _ ## _field ## _SHIFT, "4k " _desc), \
392 MRS_FIELD_VALUE(11ul<< _reg ## _ ## _field ## _SHIFT, "8k " _desc), \
393 MRS_FIELD_VALUE(12ul<< _reg ## _ ## _field ## _SHIFT, "16k " _desc), \
394 MRS_FIELD_VALUE(13ul<< _reg ## _ ## _field ## _SHIFT, "32k " _desc), \
395 MRS_FIELD_VALUE(14ul<< _reg ## _ ## _field ## _SHIFT, "64k " _desc), \
396 MRS_FIELD_VALUE(15ul<< _reg ## _ ## _field ## _SHIFT, "128k "_desc)
397
398 #define MRS_FIELD_VALUE_END { .desc = NULL }
399
400 struct mrs_field_hwcap {
401 uint64_t min;
402 u_long hwcap_val;
403 u_int hwcap_id;
404 };
405
406 #define MRS_HWCAP(_hwcap_id, _val, _min) \
407 { \
408 .hwcap_id = (_hwcap_id), \
409 .hwcap_val = (_val), \
410 .min = (_min), \
411 }
412
413 #define MRS_HWCAP_END { .hwcap_id = 0 }
414
415 struct mrs_field {
416 const char *name;
417 const struct mrs_field_value *values;
418 const struct mrs_field_hwcap *hwcaps;
419 uint64_t mask;
420 bool sign;
421 u_int type;
422 u_int width;
423 u_int shift;
424 };
425
426 #define MRS_FIELD_RES1(_width, _shift) \
427 { \
428 .sign = false, \
429 .type = MRS_EXACT | MRS_SAFE((1u << (_width)) - 1) | \
430 MRS_USERSPACE, \
431 .width = (_width), \
432 .shift = (_shift), \
433 }
434
435 #define MRS_FIELD_HWCAP(_register, _name, _sign, _type, _visibility, \
436 _values, _hwcap) \
437 { \
438 .name = #_name, \
439 .sign = (_sign), \
440 .type = ((_type) | (_visibility)), \
441 .width = _register ## _ ## _name ## _WIDTH, \
442 .shift = _register ## _ ## _name ## _SHIFT, \
443 .mask = _register ## _ ## _name ## _MASK, \
444 .values = (_values), \
445 .hwcaps = (_hwcap), \
446 }
447
448 #define MRS_FIELD(_register, _name, _sign, _type, _visibility, _values) \
449 MRS_FIELD_HWCAP(_register, _name, _sign, _type, _visibility, \
450 _values, NULL)
451
452 #define MRS_FIELD_END { .type = MRS_INVALID, }
453
454 /* CTR_EL0 */
455 static const struct mrs_field_value ctr_dic[] = {
456 MRS_FIELD_VALUE_NONE_IMPL(CTR, DIC, NONE, IMPL),
457 MRS_FIELD_VALUE_END,
458 };
459
460 static const struct mrs_field_value ctr_idc[] = {
461 MRS_FIELD_VALUE_NONE_IMPL(CTR, IDC, NONE, IMPL),
462 MRS_FIELD_VALUE_END,
463 };
464
465 static const struct mrs_field_value ctr_cwg[] = {
466 MRS_FIELD_VALUE_CACHE(CTR, CWG, "Unknown CWG",
467 "byte CWG"),
468 MRS_FIELD_VALUE_END,
469 };
470
471 static const struct mrs_field_value ctr_erg[] = {
472 MRS_FIELD_VALUE_CACHE(CTR, ERG, "Unknown ERG",
473 "byte ERG"),
474 MRS_FIELD_VALUE_END,
475 };
476
477 static const struct mrs_field_value ctr_dline[] = {
478 MRS_FIELD_VALUE_CACHE(CTR, DLINE, "4 byte D-cacheline",
479 "byte D-cacheline"),
480 MRS_FIELD_VALUE_END,
481 };
482
483 static const struct mrs_field_value ctr_l1ip[] = {
484 MRS_FIELD_VALUE(CTR_L1IP_VIPT, "VIPT I-cache"),
485 MRS_FIELD_VALUE(CTR_L1IP_PIPT, "PIPT I-cache"),
486 MRS_FIELD_VALUE_END,
487 };
488
489 static const struct mrs_field_value ctr_iline[] = {
490 MRS_FIELD_VALUE_CACHE(CTR, ILINE, "4 byte I-cacheline",
491 "byte I-cacheline"),
492 MRS_FIELD_VALUE_END,
493 };
494
495 static const struct mrs_field ctr_fields[] = {
496 /* Bit 31 is RES1 */
497 MRS_FIELD_RES1(1, 31),
498 MRS_FIELD(CTR, DIC, false, MRS_LOWER, MRS_USERSPACE, ctr_dic),
499 MRS_FIELD(CTR, IDC, false, MRS_LOWER, MRS_USERSPACE, ctr_idc),
500 MRS_FIELD(CTR, CWG, false, MRS_HIGHER_OR_ZERO, MRS_USERSPACE, ctr_cwg),
501 MRS_FIELD(CTR, ERG, false, MRS_HIGHER_OR_ZERO, MRS_USERSPACE, ctr_erg),
502 MRS_FIELD(CTR, DLINE, false, MRS_LOWER, MRS_USERSPACE, ctr_dline),
503 /* If the ICache types are different report the safe option */
504 MRS_FIELD(CTR, L1IP, false, MRS_EXACT_IF_DIFFERENT |
505 MRS_SAFE(CTR_L1IP_VIPT >> CTR_L1IP_SHIFT), MRS_USERSPACE,
506 ctr_l1ip),
507 MRS_FIELD(CTR, ILINE, false, MRS_LOWER, MRS_USERSPACE, ctr_iline),
508 MRS_FIELD_END,
509 };
510
511 /* ID_AA64AFR0_EL1 */
512 static const struct mrs_field id_aa64afr0_fields[] = {
513 MRS_FIELD_END,
514 };
515
516
517 /* ID_AA64AFR1_EL1 */
518 static const struct mrs_field id_aa64afr1_fields[] = {
519 MRS_FIELD_END,
520 };
521
522
523 /* ID_AA64DFR0_EL1 */
524 static const struct mrs_field_value id_aa64dfr0_hpmn0[] = {
525 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR0, HPMN0, CONSTR, DEFINED),
526 MRS_FIELD_VALUE_END,
527 };
528
529 static const struct mrs_field_value id_aa64dfr0_brbe[] = {
530 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR0, BRBE, NONE, IMPL),
531 MRS_FIELD_VALUE(ID_AA64DFR0_BRBE_EL3, "BRBE EL3"),
532 MRS_FIELD_VALUE_END,
533 };
534
535 static const struct mrs_field_value id_aa64dfr0_mtpmu[] = {
536 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR0, MTPMU, NONE, IMPL),
537 MRS_FIELD_VALUE(ID_AA64DFR0_MTPMU_NONE_MT_RES0, "MTPMU res0"),
538 MRS_FIELD_VALUE_END,
539 };
540
541 static const struct mrs_field_value id_aa64dfr0_tracebuffer[] = {
542 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR0, TraceBuffer, NONE, IMPL),
543 MRS_FIELD_VALUE_END,
544 };
545
546 static const struct mrs_field_value id_aa64dfr0_tracefilt[] = {
547 MRS_FIELD_VALUE(ID_AA64DFR0_TraceFilt_NONE, ""),
548 MRS_FIELD_VALUE(ID_AA64DFR0_TraceFilt_8_4, "Trace v8.4"),
549 MRS_FIELD_VALUE_END,
550 };
551
552 static const struct mrs_field_value id_aa64dfr0_doublelock[] = {
553 MRS_FIELD_VALUE(ID_AA64DFR0_DoubleLock_IMPL, "DoubleLock"),
554 MRS_FIELD_VALUE(ID_AA64DFR0_DoubleLock_NONE, ""),
555 MRS_FIELD_VALUE_END,
556 };
557
558 static const struct mrs_field_value id_aa64dfr0_pmsver[] = {
559 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_NONE, ""),
560 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE, "SPE"),
561 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE_1_1, "SPEv1p1"),
562 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE_1_2, "SPEv1p2"),
563 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE_1_3, "SPEv1p3"),
564 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE_1_4, "SPEv1p4"),
565 MRS_FIELD_VALUE_END,
566 };
567
568 static const struct mrs_field_value id_aa64dfr0_ctx_cmps[] = {
569 MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, CTX_CMPs, "CTX BKPT"),
570 MRS_FIELD_VALUE_END,
571 };
572
573 static const struct mrs_field_value id_aa64dfr0_wrps[] = {
574 MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, WRPs, "Watchpoint"),
575 MRS_FIELD_VALUE_END,
576 };
577
578 static const struct mrs_field_value id_aa64dfr0_pmss[] = {
579 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR0, PMSS, NONE, IMPL),
580 MRS_FIELD_VALUE_END,
581 };
582
583 static const struct mrs_field_value id_aa64dfr0_brps[] = {
584 MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, BRPs, "Breakpoint"),
585 MRS_FIELD_VALUE_END,
586 };
587
588 static const struct mrs_field_value id_aa64dfr0_pmuver[] = {
589 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_NONE, ""),
590 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3, "PMUv3"),
591 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_1, "PMUv3p1"),
592 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_4, "PMUv3p4"),
593 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_5, "PMUv3p5"),
594 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_7, "PMUv3p7"),
595 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_8, "PMUv3p8"),
596 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_9, "PMUv3p9"),
597 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_IMPL, "IMPL PMU"),
598 MRS_FIELD_VALUE_END,
599 };
600
601 static const struct mrs_field_value id_aa64dfr0_tracever[] = {
602 MRS_FIELD_VALUE(ID_AA64DFR0_TraceVer_NONE, ""),
603 MRS_FIELD_VALUE(ID_AA64DFR0_TraceVer_IMPL, "Trace"),
604 MRS_FIELD_VALUE_END,
605 };
606
607 static const struct mrs_field_value id_aa64dfr0_debugver[] = {
608 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8, "Debugv8"),
609 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_VHE, "Debugv8_VHE"),
610 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_2, "Debugv8p2"),
611 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_4, "Debugv8p4"),
612 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_8, "Debugv8p8"),
613 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_9, "Debugv8p9"),
614 MRS_FIELD_VALUE_END,
615 };
616
617 static const struct mrs_field id_aa64dfr0_fields[] = {
618 MRS_FIELD(ID_AA64DFR0, HPMN0, false, MRS_LOWER, 0, id_aa64dfr0_hpmn0),
619 MRS_FIELD(ID_AA64DFR0, BRBE, false, MRS_LOWER, 0, id_aa64dfr0_brbe),
620 MRS_FIELD(ID_AA64DFR0, MTPMU, true, MRS_LOWER, 0, id_aa64dfr0_mtpmu),
621 MRS_FIELD(ID_AA64DFR0, TraceBuffer, false, MRS_LOWER, 0,
622 id_aa64dfr0_tracebuffer),
623 MRS_FIELD(ID_AA64DFR0, TraceFilt, false, MRS_LOWER, 0,
624 id_aa64dfr0_tracefilt),
625 MRS_FIELD(ID_AA64DFR0, DoubleLock, true, MRS_LOWER, 0,
626 id_aa64dfr0_doublelock),
627 MRS_FIELD(ID_AA64DFR0, PMSVer, false, MRS_LOWER, 0, id_aa64dfr0_pmsver),
628 MRS_FIELD(ID_AA64DFR0, CTX_CMPs, false, MRS_LOWER, 0,
629 id_aa64dfr0_ctx_cmps),
630 MRS_FIELD(ID_AA64DFR0, WRPs, false, MRS_LOWER, MRS_USERSPACE,
631 id_aa64dfr0_wrps),
632 MRS_FIELD(ID_AA64DFR0, PMSS, false, MRS_LOWER, 0, id_aa64dfr0_pmss),
633 MRS_FIELD(ID_AA64DFR0, BRPs, false, MRS_LOWER, MRS_USERSPACE,
634 id_aa64dfr0_brps),
635 MRS_FIELD(ID_AA64DFR0, PMUVer, true, MRS_LOWER, 0, id_aa64dfr0_pmuver),
636 MRS_FIELD(ID_AA64DFR0, TraceVer, false, MRS_LOWER, 0,
637 id_aa64dfr0_tracever),
638 MRS_FIELD(ID_AA64DFR0, DebugVer, false, MRS_LOWER | MRS_SAFE(0x6), 0,
639 id_aa64dfr0_debugver),
640 MRS_FIELD_END,
641 };
642
643
644 /* ID_AA64DFR1_EL1 */
645 static const struct mrs_field_value id_aa64dfr1_dpfzs[] = {
646 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR1, DPFZS, NONE, IMPL),
647 MRS_FIELD_VALUE_END,
648 };
649
650 static const struct mrs_field_value id_aa64dfr1_pmicntr[] = {
651 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR1, PMICNTR, NONE, IMPL),
652 MRS_FIELD_VALUE_END,
653 };
654
655 static const struct mrs_field_value id_aa64dfr1_spmu[] = {
656 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR1, SPMU, NONE, IMPL),
657 MRS_FIELD_VALUE_END,
658 };
659
660 static const struct mrs_field id_aa64dfr1_fields[] = {
661 MRS_FIELD(ID_AA64DFR1, DPFZS, false, MRS_LOWER, 0, id_aa64dfr1_dpfzs),
662 MRS_FIELD(ID_AA64DFR1, PMICNTR, false, MRS_LOWER, 0, id_aa64dfr1_pmicntr),
663 MRS_FIELD(ID_AA64DFR1, SPMU, false, MRS_LOWER, 0, id_aa64dfr1_spmu),
664 MRS_FIELD_END,
665 };
666
667
668 /* ID_AA64ISAR0_EL1 */
669 static const struct mrs_field_value id_aa64isar0_rndr[] = {
670 MRS_FIELD_VALUE(ID_AA64ISAR0_RNDR_NONE, ""),
671 MRS_FIELD_VALUE(ID_AA64ISAR0_RNDR_IMPL, "RNG"),
672 MRS_FIELD_VALUE_END,
673 };
674
675 static const struct mrs_field_hwcap id_aa64isar0_rndr_caps[] = {
676 MRS_HWCAP(2, HWCAP2_RNG, ID_AA64ISAR0_RNDR_IMPL),
677 MRS_HWCAP_END
678 };
679
680 static const struct mrs_field_value id_aa64isar0_tlb[] = {
681 MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_NONE, ""),
682 MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_TLBIOS, "TLBI-OS"),
683 MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_TLBIOSR, "TLBI-OSR"),
684 MRS_FIELD_VALUE_END,
685 };
686
687 static const struct mrs_field_value id_aa64isar0_ts[] = {
688 MRS_FIELD_VALUE(ID_AA64ISAR0_TS_NONE, ""),
689 MRS_FIELD_VALUE(ID_AA64ISAR0_TS_CondM_8_4, "CondM-8.4"),
690 MRS_FIELD_VALUE(ID_AA64ISAR0_TS_CondM_8_5, "CondM-8.5"),
691 MRS_FIELD_VALUE_END,
692 };
693
694 static const struct mrs_field_hwcap id_aa64isar0_ts_caps[] = {
695 MRS_HWCAP(1, HWCAP_FLAGM, ID_AA64ISAR0_TS_CondM_8_4),
696 MRS_HWCAP(2, HWCAP2_FLAGM2, ID_AA64ISAR0_TS_CondM_8_5),
697 MRS_HWCAP_END
698 };
699
700 static const struct mrs_field_value id_aa64isar0_fhm[] = {
701 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, FHM, NONE, IMPL),
702 MRS_FIELD_VALUE_END,
703 };
704
705 static const struct mrs_field_hwcap id_aa64isar0_fhm_caps[] = {
706 MRS_HWCAP(1, HWCAP_ASIMDFHM, ID_AA64ISAR0_FHM_IMPL),
707 MRS_HWCAP_END
708 };
709
710 static const struct mrs_field_value id_aa64isar0_dp[] = {
711 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, DP, NONE, IMPL),
712 MRS_FIELD_VALUE_END,
713 };
714
715 static const struct mrs_field_hwcap id_aa64isar0_dp_caps[] = {
716 MRS_HWCAP(1, HWCAP_ASIMDDP, ID_AA64ISAR0_DP_IMPL),
717 MRS_HWCAP_END
718 };
719
720 static const struct mrs_field_value id_aa64isar0_sm4[] = {
721 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SM4, NONE, IMPL),
722 MRS_FIELD_VALUE_END,
723 };
724
725 static const struct mrs_field_hwcap id_aa64isar0_sm4_caps[] = {
726 MRS_HWCAP(1, HWCAP_SM4, ID_AA64ISAR0_SM4_IMPL),
727 MRS_HWCAP_END
728 };
729
730 static const struct mrs_field_value id_aa64isar0_sm3[] = {
731 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SM3, NONE, IMPL),
732 MRS_FIELD_VALUE_END,
733 };
734
735 static const struct mrs_field_hwcap id_aa64isar0_sm3_caps[] = {
736 MRS_HWCAP(1, HWCAP_SM3, ID_AA64ISAR0_SM3_IMPL),
737 MRS_HWCAP_END
738 };
739
740 static const struct mrs_field_value id_aa64isar0_sha3[] = {
741 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA3, NONE, IMPL),
742 MRS_FIELD_VALUE_END,
743 };
744
745 static const struct mrs_field_hwcap id_aa64isar0_sha3_caps[] = {
746 MRS_HWCAP(1, HWCAP_SHA3, ID_AA64ISAR0_SHA3_IMPL),
747 MRS_HWCAP_END
748 };
749
750 static const struct mrs_field_value id_aa64isar0_rdm[] = {
751 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, RDM, NONE, IMPL),
752 MRS_FIELD_VALUE_END,
753 };
754
755 static const struct mrs_field_hwcap id_aa64isar0_rdm_caps[] = {
756 MRS_HWCAP(1, HWCAP_ASIMDRDM, ID_AA64ISAR0_RDM_IMPL),
757 MRS_HWCAP_END
758 };
759
760 static const struct mrs_field_value id_aa64isar0_tme[] = {
761 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, TME, NONE, IMPL),
762 MRS_FIELD_VALUE_END,
763 };
764
765 static const struct mrs_field_value id_aa64isar0_atomic[] = {
766 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, Atomic, NONE, IMPL),
767 MRS_FIELD_VALUE_END,
768 };
769
770 static const struct mrs_field_hwcap id_aa64isar0_atomic_caps[] = {
771 MRS_HWCAP(1, HWCAP_ATOMICS, ID_AA64ISAR0_Atomic_IMPL),
772 MRS_HWCAP_END
773 };
774
775 static const struct mrs_field_value id_aa64isar0_crc32[] = {
776 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, CRC32, NONE, BASE),
777 MRS_FIELD_VALUE_END,
778 };
779
780 static const struct mrs_field_hwcap id_aa64isar0_crc32_caps[] = {
781 MRS_HWCAP(1, HWCAP_CRC32, ID_AA64ISAR0_CRC32_BASE),
782 MRS_HWCAP_END
783 };
784
785 static const struct mrs_field_value id_aa64isar0_sha2[] = {
786 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA2, NONE, BASE),
787 MRS_FIELD_VALUE(ID_AA64ISAR0_SHA2_512, "SHA2+SHA512"),
788 MRS_FIELD_VALUE_END,
789 };
790
791 static const struct mrs_field_hwcap id_aa64isar0_sha2_caps[] = {
792 MRS_HWCAP(1, HWCAP_SHA2, ID_AA64ISAR0_SHA2_BASE),
793 MRS_HWCAP(1, HWCAP_SHA512, ID_AA64ISAR0_SHA2_512),
794 MRS_HWCAP_END
795 };
796
797 static const struct mrs_field_value id_aa64isar0_sha1[] = {
798 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA1, NONE, BASE),
799 MRS_FIELD_VALUE_END,
800 };
801
802 static const struct mrs_field_hwcap id_aa64isar0_sha1_caps[] = {
803 MRS_HWCAP(1, HWCAP_SHA1, ID_AA64ISAR0_SHA1_BASE),
804 MRS_HWCAP_END
805 };
806
807 static const struct mrs_field_value id_aa64isar0_aes[] = {
808 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, AES, NONE, BASE),
809 MRS_FIELD_VALUE(ID_AA64ISAR0_AES_PMULL, "AES+PMULL"),
810 MRS_FIELD_VALUE_END,
811 };
812
813 static const struct mrs_field_hwcap id_aa64isar0_aes_caps[] = {
814 MRS_HWCAP(1, HWCAP_AES, ID_AA64ISAR0_AES_BASE),
815 MRS_HWCAP(1, HWCAP_PMULL, ID_AA64ISAR0_AES_PMULL),
816 MRS_HWCAP_END
817 };
818
819 static const struct mrs_field id_aa64isar0_fields[] = {
820 MRS_FIELD_HWCAP(ID_AA64ISAR0, RNDR, false, MRS_LOWER, MRS_USERSPACE,
821 id_aa64isar0_rndr, id_aa64isar0_rndr_caps),
822 MRS_FIELD(ID_AA64ISAR0, TLB, false, MRS_LOWER, 0, id_aa64isar0_tlb),
823 MRS_FIELD_HWCAP(ID_AA64ISAR0, TS, false, MRS_LOWER, MRS_USERSPACE,
824 id_aa64isar0_ts, id_aa64isar0_ts_caps),
825 MRS_FIELD_HWCAP(ID_AA64ISAR0, FHM, false, MRS_LOWER, MRS_USERSPACE,
826 id_aa64isar0_fhm, id_aa64isar0_fhm_caps),
827 MRS_FIELD_HWCAP(ID_AA64ISAR0, DP, false, MRS_LOWER, MRS_USERSPACE,
828 id_aa64isar0_dp, id_aa64isar0_dp_caps),
829 MRS_FIELD_HWCAP(ID_AA64ISAR0, SM4, false, MRS_LOWER, MRS_USERSPACE,
830 id_aa64isar0_sm4, id_aa64isar0_sm4_caps),
831 MRS_FIELD_HWCAP(ID_AA64ISAR0, SM3, false, MRS_LOWER, MRS_USERSPACE,
832 id_aa64isar0_sm3, id_aa64isar0_sm3_caps),
833 MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA3, false, MRS_LOWER, MRS_USERSPACE,
834 id_aa64isar0_sha3, id_aa64isar0_sha3_caps),
835 MRS_FIELD_HWCAP(ID_AA64ISAR0, RDM, false, MRS_LOWER, MRS_USERSPACE,
836 id_aa64isar0_rdm, id_aa64isar0_rdm_caps),
837 MRS_FIELD(ID_AA64ISAR0, TME, false, MRS_LOWER, 0, id_aa64isar0_tme),
838 MRS_FIELD_HWCAP(ID_AA64ISAR0, Atomic, false, MRS_LOWER, MRS_USERSPACE,
839 id_aa64isar0_atomic, id_aa64isar0_atomic_caps),
840 MRS_FIELD_HWCAP(ID_AA64ISAR0, CRC32, false, MRS_LOWER, MRS_USERSPACE,
841 id_aa64isar0_crc32, id_aa64isar0_crc32_caps),
842 MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA2, false, MRS_LOWER, MRS_USERSPACE,
843 id_aa64isar0_sha2, id_aa64isar0_sha2_caps),
844 MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA1, false, MRS_LOWER, MRS_USERSPACE,
845 id_aa64isar0_sha1, id_aa64isar0_sha1_caps),
846 MRS_FIELD_HWCAP(ID_AA64ISAR0, AES, false, MRS_LOWER, MRS_USERSPACE,
847 id_aa64isar0_aes, id_aa64isar0_aes_caps),
848 MRS_FIELD_END,
849 };
850
851
852 /* ID_AA64ISAR1_EL1 */
853 static const struct mrs_field_value id_aa64isar1_ls64[] = {
854 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, LS64, NONE, IMPL),
855 MRS_FIELD_VALUE(ID_AA64ISAR1_LS64_V, "LS64v"),
856 MRS_FIELD_VALUE(ID_AA64ISAR1_LS64_ACCDATA, "LS64+ACCDATA"),
857 MRS_FIELD_VALUE_END,
858 };
859
860 static const struct mrs_field_value id_aa64isar1_xs[] = {
861 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, XS, NONE, IMPL),
862 MRS_FIELD_VALUE_END,
863 };
864
865 static const struct mrs_field_value id_aa64isar1_i8mm[] = {
866 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, I8MM, NONE, IMPL),
867 MRS_FIELD_VALUE_END,
868 };
869
870 static const struct mrs_field_hwcap id_aa64isar1_i8mm_caps[] = {
871 MRS_HWCAP(2, HWCAP2_I8MM, ID_AA64ISAR1_I8MM_IMPL),
872 MRS_HWCAP_END
873 };
874
875 static const struct mrs_field_value id_aa64isar1_dgh[] = {
876 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, DGH, NONE, IMPL),
877 MRS_FIELD_VALUE_END,
878 };
879
880 static const struct mrs_field_hwcap id_aa64isar1_dgh_caps[] = {
881 MRS_HWCAP(2, HWCAP2_DGH, ID_AA64ISAR1_DGH_IMPL),
882 MRS_HWCAP_END
883 };
884
885 static const struct mrs_field_value id_aa64isar1_bf16[] = {
886 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, BF16, NONE, IMPL),
887 MRS_FIELD_VALUE(ID_AA64ISAR1_BF16_EBF, "EBF16"),
888 MRS_FIELD_VALUE_END,
889 };
890
891 static const struct mrs_field_hwcap id_aa64isar1_bf16_caps[] = {
892 MRS_HWCAP(2, HWCAP2_BF16, ID_AA64ISAR1_BF16_IMPL),
893 MRS_HWCAP_END
894 };
895
896 static const struct mrs_field_value id_aa64isar1_specres[] = {
897 MRS_FIELD_VALUE(ID_AA64ISAR1_SPECRES_NONE, ""),
898 MRS_FIELD_VALUE(ID_AA64ISAR1_SPECRES_8_5, "PredInv v8.5"),
899 MRS_FIELD_VALUE(ID_AA64ISAR1_SPECRES_8_9, "PredInv v8.9"),
900 MRS_FIELD_VALUE_END,
901 };
902
903 static const struct mrs_field_value id_aa64isar1_sb[] = {
904 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, SB, NONE, IMPL),
905 MRS_FIELD_VALUE_END,
906 };
907
908 static const struct mrs_field_hwcap id_aa64isar1_sb_caps[] = {
909 MRS_HWCAP(1, HWCAP_SB, ID_AA64ISAR1_SB_IMPL),
910 MRS_HWCAP_END
911 };
912
913 static const struct mrs_field_value id_aa64isar1_frintts[] = {
914 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, FRINTTS, NONE, IMPL),
915 MRS_FIELD_VALUE_END,
916 };
917
918 static const struct mrs_field_hwcap id_aa64isar1_frintts_caps[] = {
919 MRS_HWCAP(2, HWCAP2_FRINT, ID_AA64ISAR1_FRINTTS_IMPL),
920 MRS_HWCAP_END
921 };
922
923 static const struct mrs_field_value id_aa64isar1_gpi[] = {
924 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, GPI, NONE, IMPL),
925 MRS_FIELD_VALUE_END,
926 };
927
928 static const struct mrs_field_hwcap id_aa64isar1_gpi_caps[] = {
929 MRS_HWCAP(1, HWCAP_PACG, ID_AA64ISAR1_GPI_IMPL),
930 MRS_HWCAP_END
931 };
932
933 static const struct mrs_field_value id_aa64isar1_gpa[] = {
934 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, GPA, NONE, IMPL),
935 MRS_FIELD_VALUE_END,
936 };
937
938 static const struct mrs_field_hwcap id_aa64isar1_gpa_caps[] = {
939 MRS_HWCAP(1, HWCAP_PACG, ID_AA64ISAR1_GPA_IMPL),
940 MRS_HWCAP_END
941 };
942
943 static const struct mrs_field_value id_aa64isar1_lrcpc[] = {
944 MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_NONE, ""),
945 MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_RCPC_8_3, "RCPC-8.3"),
946 MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_RCPC_8_4, "RCPC-8.4"),
947 MRS_FIELD_VALUE_END,
948 };
949
950 static const struct mrs_field_hwcap id_aa64isar1_lrcpc_caps[] = {
951 MRS_HWCAP(1, HWCAP_LRCPC, ID_AA64ISAR1_LRCPC_RCPC_8_3),
952 MRS_HWCAP(1, HWCAP_ILRCPC, ID_AA64ISAR1_LRCPC_RCPC_8_4),
953 MRS_HWCAP_END
954 };
955
956 static const struct mrs_field_value id_aa64isar1_fcma[] = {
957 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, FCMA, NONE, IMPL),
958 MRS_FIELD_VALUE_END,
959 };
960
961 static const struct mrs_field_hwcap id_aa64isar1_fcma_caps[] = {
962 MRS_HWCAP(1, HWCAP_FCMA, ID_AA64ISAR1_FCMA_IMPL),
963 MRS_HWCAP_END
964 };
965
966 static const struct mrs_field_value id_aa64isar1_jscvt[] = {
967 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, JSCVT, NONE, IMPL),
968 MRS_FIELD_VALUE_END,
969 };
970
971 static const struct mrs_field_hwcap id_aa64isar1_jscvt_caps[] = {
972 MRS_HWCAP(1, HWCAP_JSCVT, ID_AA64ISAR1_JSCVT_IMPL),
973 MRS_HWCAP_END
974 };
975
976 static const struct mrs_field_value id_aa64isar1_api[] = {
977 MRS_FIELD_VALUE(ID_AA64ISAR1_API_NONE, ""),
978 MRS_FIELD_VALUE(ID_AA64ISAR1_API_PAC, "API PAC"),
979 MRS_FIELD_VALUE(ID_AA64ISAR1_API_EPAC, "API EPAC"),
980 MRS_FIELD_VALUE(ID_AA64ISAR1_API_EPAC2, "Impl PAuth+EPAC2"),
981 MRS_FIELD_VALUE(ID_AA64ISAR1_API_FPAC, "Impl PAuth+FPAC"),
982 MRS_FIELD_VALUE(ID_AA64ISAR1_API_FPAC_COMBINED,
983 "Impl PAuth+FPAC+Combined"),
984 MRS_FIELD_VALUE_END,
985 };
986
987 static const struct mrs_field_hwcap id_aa64isar1_api_caps[] = {
988 MRS_HWCAP(1, HWCAP_PACA, ID_AA64ISAR1_API_PAC),
989 MRS_HWCAP_END
990 };
991
992 static const struct mrs_field_value id_aa64isar1_apa[] = {
993 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_NONE, ""),
994 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_PAC, "APA PAC"),
995 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_EPAC, "APA EPAC"),
996 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_EPAC2, "APA EPAC2"),
997 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_FPAC, "APA FPAC"),
998 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_FPAC_COMBINED,
999 "APA FPAC+Combined"),
1000 MRS_FIELD_VALUE_END,
1001 };
1002
1003 static const struct mrs_field_hwcap id_aa64isar1_apa_caps[] = {
1004 MRS_HWCAP(1, HWCAP_PACA, ID_AA64ISAR1_APA_PAC),
1005 MRS_HWCAP_END
1006 };
1007
1008 static const struct mrs_field_value id_aa64isar1_dpb[] = {
1009 MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_NONE, ""),
1010 MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_DCCVAP, "DCPoP"),
1011 MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_DCCVADP, "DCCVADP"),
1012 MRS_FIELD_VALUE_END,
1013 };
1014
1015 static const struct mrs_field_hwcap id_aa64isar1_dpb_caps[] = {
1016 MRS_HWCAP(1, HWCAP_DCPOP, ID_AA64ISAR1_DPB_DCCVAP),
1017 MRS_HWCAP(2, HWCAP2_DCPODP, ID_AA64ISAR1_DPB_DCCVADP),
1018 MRS_HWCAP_END
1019 };
1020
1021 static const struct mrs_field id_aa64isar1_fields[] = {
1022 MRS_FIELD(ID_AA64ISAR1, LS64, false, MRS_LOWER, 0, id_aa64isar1_ls64),
1023 MRS_FIELD(ID_AA64ISAR1, XS, false, MRS_LOWER, 0, id_aa64isar1_xs),
1024 MRS_FIELD_HWCAP(ID_AA64ISAR1, I8MM, false, MRS_LOWER, MRS_USERSPACE,
1025 id_aa64isar1_i8mm, id_aa64isar1_i8mm_caps),
1026 MRS_FIELD_HWCAP(ID_AA64ISAR1, DGH, false, MRS_LOWER, MRS_USERSPACE,
1027 id_aa64isar1_dgh, id_aa64isar1_dgh_caps),
1028 MRS_FIELD_HWCAP(ID_AA64ISAR1, BF16, false, MRS_LOWER, MRS_USERSPACE,
1029 id_aa64isar1_bf16, id_aa64isar1_bf16_caps),
1030 MRS_FIELD(ID_AA64ISAR1, SPECRES, false, MRS_LOWER, 0,
1031 id_aa64isar1_specres),
1032 MRS_FIELD_HWCAP(ID_AA64ISAR1, SB, false, MRS_LOWER, MRS_USERSPACE,
1033 id_aa64isar1_sb, id_aa64isar1_sb_caps),
1034 MRS_FIELD_HWCAP(ID_AA64ISAR1, FRINTTS, false, MRS_LOWER, MRS_USERSPACE,
1035 id_aa64isar1_frintts, id_aa64isar1_frintts_caps),
1036 MRS_FIELD_HWCAP(ID_AA64ISAR1, GPI, false, MRS_LOWER, MRS_USERSPACE,
1037 id_aa64isar1_gpi, id_aa64isar1_gpi_caps),
1038 MRS_FIELD_HWCAP(ID_AA64ISAR1, GPA, false, MRS_LOWER, MRS_USERSPACE,
1039 id_aa64isar1_gpa, id_aa64isar1_gpa_caps),
1040 MRS_FIELD_HWCAP(ID_AA64ISAR1, LRCPC, false, MRS_LOWER, MRS_USERSPACE,
1041 id_aa64isar1_lrcpc, id_aa64isar1_lrcpc_caps),
1042 MRS_FIELD_HWCAP(ID_AA64ISAR1, FCMA, false, MRS_LOWER, MRS_USERSPACE,
1043 id_aa64isar1_fcma, id_aa64isar1_fcma_caps),
1044 MRS_FIELD_HWCAP(ID_AA64ISAR1, JSCVT, false, MRS_LOWER, MRS_USERSPACE,
1045 id_aa64isar1_jscvt, id_aa64isar1_jscvt_caps),
1046 MRS_FIELD_HWCAP(ID_AA64ISAR1, API, false, MRS_LOWER, MRS_USERSPACE,
1047 id_aa64isar1_api, id_aa64isar1_api_caps),
1048 MRS_FIELD_HWCAP(ID_AA64ISAR1, APA, false, MRS_LOWER, MRS_USERSPACE,
1049 id_aa64isar1_apa, id_aa64isar1_apa_caps),
1050 MRS_FIELD_HWCAP(ID_AA64ISAR1, DPB, false, MRS_LOWER, MRS_USERSPACE,
1051 id_aa64isar1_dpb, id_aa64isar1_dpb_caps),
1052 MRS_FIELD_END,
1053 };
1054
1055
1056 /* ID_AA64ISAR2_EL1 */
1057 static const struct mrs_field_value id_aa64isar2_ats1a[] = {
1058 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, ATS1A, NONE, IMPL),
1059 MRS_FIELD_VALUE_END,
1060 };
1061
1062 static const struct mrs_field_value id_aa64isar2_cssc[] = {
1063 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, CSSC, NONE, IMPL),
1064 MRS_FIELD_VALUE_END,
1065 };
1066
1067 static const struct mrs_field_value id_aa64isar2_rprfm[] = {
1068 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, RPRFM, NONE, IMPL),
1069 MRS_FIELD_VALUE_END,
1070 };
1071
1072 static const struct mrs_field_value id_aa64isar2_prfmslc[] = {
1073 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, PRFMSLC, NONE, IMPL),
1074 MRS_FIELD_VALUE_END,
1075 };
1076
1077 static const struct mrs_field_value id_aa64isar2_clrbhb[] = {
1078 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, CLRBHB, NONE, IMPL),
1079 MRS_FIELD_VALUE_END,
1080 };
1081
1082 static const struct mrs_field_value id_aa64isar2_pac_frac[] = {
1083 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, PAC_frac, NONE, IMPL),
1084 MRS_FIELD_VALUE_END,
1085 };
1086
1087 static const struct mrs_field_value id_aa64isar2_bc[] = {
1088 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, BC, NONE, IMPL),
1089 MRS_FIELD_VALUE_END,
1090 };
1091
1092 static const struct mrs_field_value id_aa64isar2_mops[] = {
1093 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, MOPS, NONE, IMPL),
1094 MRS_FIELD_VALUE_END,
1095 };
1096
1097 static const struct mrs_field_hwcap id_aa64isar2_mops_caps[] = {
1098 MRS_HWCAP(2, HWCAP2_MOPS, ID_AA64ISAR2_MOPS_IMPL),
1099 MRS_HWCAP_END
1100 };
1101
1102 static const struct mrs_field_value id_aa64isar2_apa3[] = {
1103 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_NONE, ""),
1104 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_PAC, "APA3 PAC"),
1105 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_EPAC, "APA3 EPAC"),
1106 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_EPAC2, "APA3 EPAC2"),
1107 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_FPAC, "APA3 FPAC"),
1108 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_FPAC_COMBINED,
1109 "APA3 FPAC+Combined"),
1110 MRS_FIELD_VALUE_END,
1111 };
1112
1113 static const struct mrs_field_hwcap id_aa64isar2_apa3_caps[] = {
1114 MRS_HWCAP(1, HWCAP_PACA, ID_AA64ISAR2_APA3_PAC),
1115 MRS_HWCAP_END
1116 };
1117
1118 static const struct mrs_field_value id_aa64isar2_gpa3[] = {
1119 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, GPA3, NONE, IMPL),
1120 MRS_FIELD_VALUE_END,
1121 };
1122
1123 static const struct mrs_field_hwcap id_aa64isar2_gpa3_caps[] = {
1124 MRS_HWCAP(1, HWCAP_PACG, ID_AA64ISAR2_GPA3_IMPL),
1125 MRS_HWCAP_END
1126 };
1127
1128 static const struct mrs_field_value id_aa64isar2_rpres[] = {
1129 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, RPRES, NONE, IMPL),
1130 MRS_FIELD_VALUE_END,
1131 };
1132
1133 static const struct mrs_field_hwcap id_aa64isar2_rpres_caps[] = {
1134 MRS_HWCAP(2, HWCAP2_RPRES, ID_AA64ISAR2_RPRES_IMPL),
1135 MRS_HWCAP_END
1136 };
1137
1138 static const struct mrs_field_value id_aa64isar2_wfxt[] = {
1139 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, WFxT, NONE, IMPL),
1140 MRS_FIELD_VALUE_END,
1141 };
1142
1143 static const struct mrs_field_hwcap id_aa64isar2_wfxt_caps[] = {
1144 MRS_HWCAP(2, HWCAP2_WFXT, ID_AA64ISAR2_WFxT_IMPL),
1145 MRS_HWCAP_END
1146 };
1147
1148 static const struct mrs_field id_aa64isar2_fields[] = {
1149 MRS_FIELD(ID_AA64ISAR2, ATS1A, false, MRS_LOWER, 0, id_aa64isar2_ats1a),
1150 MRS_FIELD(ID_AA64ISAR2, CSSC, false, MRS_LOWER, 0, id_aa64isar2_cssc),
1151 MRS_FIELD(ID_AA64ISAR2, RPRFM, false, MRS_LOWER, 0, id_aa64isar2_rprfm),
1152 MRS_FIELD(ID_AA64ISAR2, PRFMSLC, false, MRS_LOWER, 0, id_aa64isar2_prfmslc),
1153 MRS_FIELD(ID_AA64ISAR2, CLRBHB, false, MRS_LOWER, 0, id_aa64isar2_clrbhb),
1154 MRS_FIELD(ID_AA64ISAR2, PAC_frac, false, MRS_LOWER, 0,
1155 id_aa64isar2_pac_frac),
1156 MRS_FIELD(ID_AA64ISAR2, BC, false, MRS_LOWER, 0, id_aa64isar2_bc),
1157 MRS_FIELD_HWCAP(ID_AA64ISAR2, MOPS, false, MRS_LOWER, MRS_USERSPACE,
1158 id_aa64isar2_mops, id_aa64isar2_mops_caps),
1159 MRS_FIELD_HWCAP(ID_AA64ISAR2, APA3, false, MRS_LOWER, MRS_USERSPACE,
1160 id_aa64isar2_apa3, id_aa64isar2_apa3_caps),
1161 MRS_FIELD_HWCAP(ID_AA64ISAR2, GPA3, false, MRS_LOWER, MRS_USERSPACE,
1162 id_aa64isar2_gpa3, id_aa64isar2_gpa3_caps),
1163 MRS_FIELD_HWCAP(ID_AA64ISAR2, RPRES, false, MRS_LOWER, MRS_USERSPACE,
1164 id_aa64isar2_rpres, id_aa64isar2_rpres_caps),
1165 MRS_FIELD_HWCAP(ID_AA64ISAR2, WFxT, false, MRS_LOWER, 0,
1166 id_aa64isar2_wfxt, id_aa64isar2_wfxt_caps),
1167 MRS_FIELD_END,
1168 };
1169
1170
1171 /* ID_AA64MMFR0_EL1 */
1172 static const struct mrs_field_value id_aa64mmfr0_ecv[] = {
1173 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, ECV, NONE, IMPL),
1174 MRS_FIELD_VALUE(ID_AA64MMFR0_ECV_POFF, "ECV POFF"),
1175 MRS_FIELD_VALUE_END,
1176 };
1177
1178 static const struct mrs_field_value id_aa64mmfr0_fgt[] = {
1179 MRS_FIELD_VALUE(ID_AA64MMFR0_FGT_NONE, ""),
1180 MRS_FIELD_VALUE(ID_AA64MMFR0_FGT_8_6, "FGT v8.6"),
1181 MRS_FIELD_VALUE(ID_AA64MMFR0_FGT_8_9, "FGT v8.9"),
1182 MRS_FIELD_VALUE_END,
1183 };
1184
1185 static const struct mrs_field_value id_aa64mmfr0_exs[] = {
1186 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, ExS, ALL, IMPL),
1187 MRS_FIELD_VALUE_END,
1188 };
1189
1190 static const struct mrs_field_value id_aa64mmfr0_tgran4_2[] = {
1191 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_TGran4, ""),
1192 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_NONE, "No S2 TGran4"),
1193 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_IMPL, "S2 TGran4"),
1194 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_LPA2, "S2 TGran4+LPA2"),
1195 MRS_FIELD_VALUE_END,
1196 };
1197
1198 static const struct mrs_field_value id_aa64mmfr0_tgran64_2[] = {
1199 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_TGran64, ""),
1200 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_NONE, "No S2 TGran64"),
1201 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_IMPL, "S2 TGran64"),
1202 MRS_FIELD_VALUE_END,
1203 };
1204
1205 static const struct mrs_field_value id_aa64mmfr0_tgran16_2[] = {
1206 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_TGran16, ""),
1207 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_NONE, "No S2 TGran16"),
1208 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_IMPL, "S2 TGran16"),
1209 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_LPA2, "S2 TGran16+LPA2"),
1210 MRS_FIELD_VALUE_END,
1211 };
1212
1213 static const struct mrs_field_value id_aa64mmfr0_tgran4[] = {
1214 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran4, NONE, IMPL),
1215 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_LPA2, "TGran4+LPA2"),
1216 MRS_FIELD_VALUE_END,
1217 };
1218
1219 static const struct mrs_field_value id_aa64mmfr0_tgran64[] = {
1220 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran64, NONE, IMPL),
1221 MRS_FIELD_VALUE_END,
1222 };
1223
1224 static const struct mrs_field_value id_aa64mmfr0_tgran16[] = {
1225 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran16, NONE, IMPL),
1226 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_LPA2, "TGran16+LPA2"),
1227 MRS_FIELD_VALUE_END,
1228 };
1229
1230 static const struct mrs_field_value id_aa64mmfr0_bigendel0[] = {
1231 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, BigEndEL0, FIXED, MIXED),
1232 MRS_FIELD_VALUE_END,
1233 };
1234
1235 static const struct mrs_field_value id_aa64mmfr0_snsmem[] = {
1236 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, SNSMem, NONE, DISTINCT),
1237 MRS_FIELD_VALUE_END,
1238 };
1239
1240 static const struct mrs_field_value id_aa64mmfr0_bigend[] = {
1241 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, BigEnd, FIXED, MIXED),
1242 MRS_FIELD_VALUE_END,
1243 };
1244
1245 static const struct mrs_field_value id_aa64mmfr0_asidbits[] = {
1246 MRS_FIELD_VALUE(ID_AA64MMFR0_ASIDBits_8, "8bit ASID"),
1247 MRS_FIELD_VALUE(ID_AA64MMFR0_ASIDBits_16, "16bit ASID"),
1248 MRS_FIELD_VALUE_END,
1249 };
1250
1251 static const struct mrs_field_value id_aa64mmfr0_parange[] = {
1252 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4G, "4GB PA"),
1253 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_64G, "64GB PA"),
1254 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_1T, "1TB PA"),
1255 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4T, "4TB PA"),
1256 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_16T, "16TB PA"),
1257 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_256T, "256TB PA"),
1258 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4P, "4PB PA"),
1259 MRS_FIELD_VALUE_END,
1260 };
1261
1262 static const struct mrs_field id_aa64mmfr0_fields[] = {
1263 MRS_FIELD(ID_AA64MMFR0, ECV, false, MRS_LOWER, 0, id_aa64mmfr0_ecv),
1264 MRS_FIELD(ID_AA64MMFR0, FGT, false, MRS_LOWER, 0, id_aa64mmfr0_fgt),
1265 MRS_FIELD(ID_AA64MMFR0, ExS, false, MRS_LOWER, 0, id_aa64mmfr0_exs),
1266 MRS_FIELD(ID_AA64MMFR0, TGran4_2, false, MRS_LOWER, 0,
1267 id_aa64mmfr0_tgran4_2),
1268 MRS_FIELD(ID_AA64MMFR0, TGran64_2, false, MRS_LOWER, 0,
1269 id_aa64mmfr0_tgran64_2),
1270 MRS_FIELD(ID_AA64MMFR0, TGran16_2, false, MRS_LOWER, 0,
1271 id_aa64mmfr0_tgran16_2),
1272 MRS_FIELD(ID_AA64MMFR0, TGran4, true, MRS_LOWER, 0,
1273 id_aa64mmfr0_tgran4),
1274 MRS_FIELD(ID_AA64MMFR0, TGran64, true, MRS_LOWER, 0,
1275 id_aa64mmfr0_tgran64),
1276 MRS_FIELD(ID_AA64MMFR0, TGran16, false, MRS_LOWER, 0,
1277 id_aa64mmfr0_tgran16),
1278 MRS_FIELD(ID_AA64MMFR0, BigEndEL0, false, MRS_LOWER, 0,
1279 id_aa64mmfr0_bigendel0),
1280 MRS_FIELD(ID_AA64MMFR0, SNSMem, false, MRS_LOWER, 0,
1281 id_aa64mmfr0_snsmem),
1282 MRS_FIELD(ID_AA64MMFR0, BigEnd, false, MRS_LOWER, 0,
1283 id_aa64mmfr0_bigend),
1284 MRS_FIELD(ID_AA64MMFR0, ASIDBits, false, MRS_LOWER, 0,
1285 id_aa64mmfr0_asidbits),
1286 MRS_FIELD(ID_AA64MMFR0, PARange, false, MRS_LOWER, 0,
1287 id_aa64mmfr0_parange),
1288 MRS_FIELD_END,
1289 };
1290
1291
1292 /* ID_AA64MMFR1_EL1 */
1293 static const struct mrs_field_value id_aa64mmfr1_ecbhb[] = {
1294 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, ECBHB, NONE, IMPL),
1295 MRS_FIELD_VALUE_END,
1296 };
1297
1298 static const struct mrs_field_value id_aa64mmfr1_cmovw[] = {
1299 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, CMOVW, NONE, IMPL),
1300 MRS_FIELD_VALUE_END,
1301 };
1302
1303 static const struct mrs_field_value id_aa64mmfr1_tidcp1[] = {
1304 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, TIDCP1, NONE, IMPL),
1305 MRS_FIELD_VALUE_END,
1306 };
1307
1308 static const struct mrs_field_value id_aa64mmfr1_ntlbpa[] = {
1309 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, nTLBPA, NONE, IMPL),
1310 MRS_FIELD_VALUE_END,
1311 };
1312
1313 static const struct mrs_field_value id_aa64mmfr1_afp[] = {
1314 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, AFP, NONE, IMPL),
1315 MRS_FIELD_VALUE_END,
1316 };
1317
1318 static const struct mrs_field_hwcap id_aa64mmfr1_afp_caps[] = {
1319 MRS_HWCAP(2, HWCAP2_AFP, ID_AA64MMFR1_AFP_IMPL),
1320 MRS_HWCAP_END
1321 };
1322
1323 static const struct mrs_field_value id_aa64mmfr1_hcx[] = {
1324 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, HCX, NONE, IMPL),
1325 MRS_FIELD_VALUE_END,
1326 };
1327
1328 static const struct mrs_field_value id_aa64mmfr1_ets[] = {
1329 MRS_FIELD_VALUE(ID_AA64MMFR1_ETS_NONE, ""),
1330 MRS_FIELD_VALUE(ID_AA64MMFR1_ETS_NONE2, ""),
1331 MRS_FIELD_VALUE(ID_AA64MMFR1_ETS_IMPL, "ETS2"),
1332 MRS_FIELD_VALUE_END,
1333 };
1334
1335 static const struct mrs_field_value id_aa64mmfr1_twed[] = {
1336 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, TWED, NONE, IMPL),
1337 MRS_FIELD_VALUE_END,
1338 };
1339
1340 static const struct mrs_field_value id_aa64mmfr1_xnx[] = {
1341 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, XNX, NONE, IMPL),
1342 MRS_FIELD_VALUE_END,
1343 };
1344
1345 static const struct mrs_field_value id_aa64mmfr1_specsei[] = {
1346 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, SpecSEI, NONE, IMPL),
1347 MRS_FIELD_VALUE_END,
1348 };
1349
1350 static const struct mrs_field_value id_aa64mmfr1_pan[] = {
1351 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, PAN, NONE, IMPL),
1352 MRS_FIELD_VALUE(ID_AA64MMFR1_PAN_ATS1E1, "PAN+ATS1E1"),
1353 MRS_FIELD_VALUE(ID_AA64MMFR1_PAN_EPAN, "EPAN"),
1354 MRS_FIELD_VALUE_END,
1355 };
1356
1357 static const struct mrs_field_value id_aa64mmfr1_lo[] = {
1358 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, LO, NONE, IMPL),
1359 MRS_FIELD_VALUE_END,
1360 };
1361
1362 static const struct mrs_field_value id_aa64mmfr1_hpds[] = {
1363 MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_NONE, ""),
1364 MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_HPD, "HPD"),
1365 MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_TTPBHA, "HPD+TTPBHA"),
1366 MRS_FIELD_VALUE_END,
1367 };
1368
1369 static const struct mrs_field_value id_aa64mmfr1_vh[] = {
1370 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, VH, NONE, IMPL),
1371 MRS_FIELD_VALUE_END,
1372 };
1373
1374 static const struct mrs_field_value id_aa64mmfr1_vmidbits[] = {
1375 MRS_FIELD_VALUE(ID_AA64MMFR1_VMIDBits_8, "8bit VMID"),
1376 MRS_FIELD_VALUE(ID_AA64MMFR1_VMIDBits_16, "16bit VMID"),
1377 MRS_FIELD_VALUE_END,
1378 };
1379
1380 static const struct mrs_field_value id_aa64mmfr1_hafdbs[] = {
1381 MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_NONE, ""),
1382 MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_AF, "HAF"),
1383 MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_AF_DBS, "HAF+DS"),
1384 MRS_FIELD_VALUE_END,
1385 };
1386
1387 static const struct mrs_field id_aa64mmfr1_fields[] = {
1388 MRS_FIELD(ID_AA64MMFR1, ECBHB, false, MRS_LOWER, 0, id_aa64mmfr1_ecbhb),
1389 MRS_FIELD(ID_AA64MMFR1, CMOVW, false, MRS_LOWER, 0, id_aa64mmfr1_cmovw),
1390 MRS_FIELD(ID_AA64MMFR1, TIDCP1, false, MRS_LOWER, 0,
1391 id_aa64mmfr1_tidcp1),
1392 MRS_FIELD(ID_AA64MMFR1, nTLBPA, false, MRS_LOWER, 0,
1393 id_aa64mmfr1_ntlbpa),
1394 MRS_FIELD_HWCAP(ID_AA64MMFR1, AFP, false, MRS_LOWER, 0,
1395 id_aa64mmfr1_afp, id_aa64mmfr1_afp_caps),
1396 MRS_FIELD(ID_AA64MMFR1, HCX, false, MRS_LOWER, 0, id_aa64mmfr1_hcx),
1397 MRS_FIELD(ID_AA64MMFR1, ETS, false, MRS_LOWER, 0, id_aa64mmfr1_ets),
1398 MRS_FIELD(ID_AA64MMFR1, TWED, false, MRS_LOWER, 0, id_aa64mmfr1_twed),
1399 MRS_FIELD(ID_AA64MMFR1, XNX, false, MRS_LOWER, 0, id_aa64mmfr1_xnx),
1400 /*
1401 * SpecSEI != 0 indicates the CPU might generate an external abort
1402 * under speculation, while 0 indicates it can't happen. It's safer
1403 * to incorrectly indicate it might happen when it can't rather than
1404 * say it can't happen when it could. As such use the largest value
1405 * found in the system.
1406 */
1407 MRS_FIELD(ID_AA64MMFR1, SpecSEI, false, MRS_HIGHER, 0,
1408 id_aa64mmfr1_specsei),
1409 MRS_FIELD(ID_AA64MMFR1, PAN, false, MRS_LOWER, 0, id_aa64mmfr1_pan),
1410 MRS_FIELD(ID_AA64MMFR1, LO, false, MRS_LOWER, 0, id_aa64mmfr1_lo),
1411 MRS_FIELD(ID_AA64MMFR1, HPDS, false, MRS_LOWER, 0, id_aa64mmfr1_hpds),
1412 MRS_FIELD(ID_AA64MMFR1, VH, false, MRS_LOWER, 0, id_aa64mmfr1_vh),
1413 MRS_FIELD(ID_AA64MMFR1, VMIDBits, false, MRS_LOWER, 0,
1414 id_aa64mmfr1_vmidbits),
1415 MRS_FIELD(ID_AA64MMFR1, HAFDBS, false, MRS_LOWER, 0, id_aa64mmfr1_hafdbs),
1416 MRS_FIELD_END,
1417 };
1418
1419
1420 /* ID_AA64MMFR2_EL1 */
1421 static const struct mrs_field_value id_aa64mmfr2_e0pd[] = {
1422 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, E0PD, NONE, IMPL),
1423 MRS_FIELD_VALUE_END,
1424 };
1425
1426 static const struct mrs_field_value id_aa64mmfr2_evt[] = {
1427 MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_NONE, ""),
1428 MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_8_2, "EVT-8.2"),
1429 MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_8_5, "EVT-8.5"),
1430 MRS_FIELD_VALUE_END,
1431 };
1432
1433 static const struct mrs_field_value id_aa64mmfr2_bbm[] = {
1434 MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL0, ""),
1435 MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL1, "BBM level 1"),
1436 MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL2, "BBM level 2"),
1437 MRS_FIELD_VALUE_END,
1438 };
1439
1440 static const struct mrs_field_value id_aa64mmfr2_ttl[] = {
1441 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, TTL, NONE, IMPL),
1442 MRS_FIELD_VALUE_END,
1443 };
1444
1445 static const struct mrs_field_value id_aa64mmfr2_fwb[] = {
1446 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, FWB, NONE, IMPL),
1447 MRS_FIELD_VALUE_END,
1448 };
1449
1450 static const struct mrs_field_value id_aa64mmfr2_ids[] = {
1451 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, IDS, NONE, IMPL),
1452 MRS_FIELD_VALUE_END,
1453 };
1454
1455 static const struct mrs_field_value id_aa64mmfr2_at[] = {
1456 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, AT, NONE, IMPL),
1457 MRS_FIELD_VALUE_END,
1458 };
1459
1460 static const struct mrs_field_hwcap id_aa64mmfr2_at_caps[] = {
1461 MRS_HWCAP(1, HWCAP_USCAT, ID_AA64MMFR2_AT_IMPL),
1462 MRS_HWCAP_END
1463 };
1464
1465 static const struct mrs_field_value id_aa64mmfr2_st[] = {
1466 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, ST, NONE, IMPL),
1467 MRS_FIELD_VALUE_END,
1468 };
1469
1470 static const struct mrs_field_value id_aa64mmfr2_nv[] = {
1471 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, NV, NONE, 8_3),
1472 MRS_FIELD_VALUE(ID_AA64MMFR2_NV_8_4, "NV v8.4"),
1473 MRS_FIELD_VALUE_END,
1474 };
1475
1476 static const struct mrs_field_value id_aa64mmfr2_ccidx[] = {
1477 MRS_FIELD_VALUE(ID_AA64MMFR2_CCIDX_32, "32bit CCIDX"),
1478 MRS_FIELD_VALUE(ID_AA64MMFR2_CCIDX_64, "64bit CCIDX"),
1479 MRS_FIELD_VALUE_END,
1480 };
1481
1482 static const struct mrs_field_value id_aa64mmfr2_varange[] = {
1483 MRS_FIELD_VALUE(ID_AA64MMFR2_VARange_48, "48bit VA"),
1484 MRS_FIELD_VALUE(ID_AA64MMFR2_VARange_52, "52bit VA"),
1485 MRS_FIELD_VALUE_END,
1486 };
1487
1488 static const struct mrs_field_value id_aa64mmfr2_iesb[] = {
1489 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, IESB, NONE, IMPL),
1490 MRS_FIELD_VALUE_END,
1491 };
1492
1493 static const struct mrs_field_value id_aa64mmfr2_lsm[] = {
1494 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, LSM, NONE, IMPL),
1495 MRS_FIELD_VALUE_END,
1496 };
1497
1498 static const struct mrs_field_value id_aa64mmfr2_uao[] = {
1499 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, UAO, NONE, IMPL),
1500 MRS_FIELD_VALUE_END,
1501 };
1502
1503 static const struct mrs_field_value id_aa64mmfr2_cnp[] = {
1504 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, CnP, NONE, IMPL),
1505 MRS_FIELD_VALUE_END,
1506 };
1507
1508 static const struct mrs_field id_aa64mmfr2_fields[] = {
1509 MRS_FIELD(ID_AA64MMFR2, E0PD, false, MRS_LOWER, 0, id_aa64mmfr2_e0pd),
1510 MRS_FIELD(ID_AA64MMFR2, EVT, false, MRS_LOWER, 0, id_aa64mmfr2_evt),
1511 MRS_FIELD(ID_AA64MMFR2, BBM, false, MRS_LOWER, 0, id_aa64mmfr2_bbm),
1512 MRS_FIELD(ID_AA64MMFR2, TTL, false, MRS_LOWER, 0, id_aa64mmfr2_ttl),
1513 MRS_FIELD(ID_AA64MMFR2, FWB, false, MRS_LOWER, 0, id_aa64mmfr2_fwb),
1514 MRS_FIELD(ID_AA64MMFR2, IDS, false, MRS_LOWER, 0, id_aa64mmfr2_ids),
1515 MRS_FIELD_HWCAP(ID_AA64MMFR2, AT, false, MRS_LOWER, MRS_USERSPACE,
1516 id_aa64mmfr2_at, id_aa64mmfr2_at_caps),
1517 MRS_FIELD(ID_AA64MMFR2, ST, false, MRS_LOWER, 0, id_aa64mmfr2_st),
1518 MRS_FIELD(ID_AA64MMFR2, NV, false, MRS_LOWER, 0, id_aa64mmfr2_nv),
1519 MRS_FIELD(ID_AA64MMFR2, CCIDX, false, MRS_LOWER, 0, id_aa64mmfr2_ccidx),
1520 MRS_FIELD(ID_AA64MMFR2, VARange, false, MRS_LOWER, 0,
1521 id_aa64mmfr2_varange),
1522 MRS_FIELD(ID_AA64MMFR2, IESB, false, MRS_LOWER, 0, id_aa64mmfr2_iesb),
1523 MRS_FIELD(ID_AA64MMFR2, LSM, false, MRS_LOWER, 0, id_aa64mmfr2_lsm),
1524 MRS_FIELD(ID_AA64MMFR2, UAO, false, MRS_LOWER, 0, id_aa64mmfr2_uao),
1525 MRS_FIELD(ID_AA64MMFR2, CnP, false, MRS_LOWER, 0, id_aa64mmfr2_cnp),
1526 MRS_FIELD_END,
1527 };
1528
1529
1530 /* ID_AA64MMFR2_EL1 */
1531 static const struct mrs_field_value id_aa64mmfr3_spec_fpacc[] = {
1532 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, Spec_FPACC, NONE, IMPL),
1533 MRS_FIELD_VALUE_END,
1534 };
1535
1536 static const struct mrs_field_value id_aa64mmfr3_aderr[] = {
1537 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, ADERR, NONE, SOME),
1538 MRS_FIELD_VALUE_END,
1539 };
1540
1541 static const struct mrs_field_value id_aa64mmfr3_sderr[] = {
1542 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, SDERR, NONE, ALL),
1543 MRS_FIELD_VALUE_END,
1544 };
1545
1546 static const struct mrs_field_value id_aa64mmfr3_anerr[] = {
1547 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, ANERR, NONE, SOME),
1548 MRS_FIELD_VALUE_END,
1549 };
1550
1551 static const struct mrs_field_value id_aa64mmfr3_snerr[] = {
1552 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, SNERR, NONE, ALL),
1553 MRS_FIELD_VALUE_END,
1554 };
1555
1556 static const struct mrs_field_value id_aa64mmfr3_mec[] = {
1557 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, MEC, NONE, IMPL),
1558 MRS_FIELD_VALUE_END,
1559 };
1560
1561 static const struct mrs_field_value id_aa64mmfr3_aie[] = {
1562 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, AIE, NONE, IMPL),
1563 MRS_FIELD_VALUE_END,
1564 };
1565
1566 static const struct mrs_field_value id_aa64mmfr3_s2poe[] = {
1567 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, S2POE, NONE, IMPL),
1568 MRS_FIELD_VALUE_END,
1569 };
1570
1571 static const struct mrs_field_value id_aa64mmfr3_s1poe[] = {
1572 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, S1POE, NONE, IMPL),
1573 MRS_FIELD_VALUE_END,
1574 };
1575
1576 static const struct mrs_field_value id_aa64mmfr3_s2pie[] = {
1577 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, S2PIE, NONE, IMPL),
1578 MRS_FIELD_VALUE_END,
1579 };
1580
1581 static const struct mrs_field_value id_aa64mmfr3_s1pie[] = {
1582 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, S1PIE, NONE, IMPL),
1583 MRS_FIELD_VALUE_END,
1584 };
1585
1586 static const struct mrs_field_value id_aa64mmfr3_sctlrx[] = {
1587 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, SCTLRX, NONE, IMPL),
1588 MRS_FIELD_VALUE_END,
1589 };
1590
1591 static const struct mrs_field_value id_aa64mmfr3_tcrx[] = {
1592 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, TCRX, NONE, IMPL),
1593 MRS_FIELD_VALUE_END,
1594 };
1595
1596 static const struct mrs_field id_aa64mmfr3_fields[] = {
1597 MRS_FIELD(ID_AA64MMFR3, Spec_FPACC, false, MRS_LOWER, 0,
1598 id_aa64mmfr3_spec_fpacc),
1599 MRS_FIELD(ID_AA64MMFR3, ADERR, false, MRS_LOWER, 0, id_aa64mmfr3_aderr),
1600 MRS_FIELD(ID_AA64MMFR3, SDERR, false, MRS_LOWER, 0, id_aa64mmfr3_sderr),
1601 MRS_FIELD(ID_AA64MMFR3, ANERR, false, MRS_LOWER, 0, id_aa64mmfr3_anerr),
1602 MRS_FIELD(ID_AA64MMFR3, SNERR, false, MRS_LOWER, 0, id_aa64mmfr3_snerr),
1603 MRS_FIELD(ID_AA64MMFR3, MEC, false, MRS_LOWER, 0, id_aa64mmfr3_mec),
1604 MRS_FIELD(ID_AA64MMFR3, AIE, false, MRS_LOWER, 0, id_aa64mmfr3_aie),
1605 MRS_FIELD(ID_AA64MMFR3, S2POE, false, MRS_LOWER, 0, id_aa64mmfr3_s2poe),
1606 MRS_FIELD(ID_AA64MMFR3, S1POE, false, MRS_LOWER, 0, id_aa64mmfr3_s1poe),
1607 MRS_FIELD(ID_AA64MMFR3, S2PIE, false, MRS_LOWER, 0, id_aa64mmfr3_s2pie),
1608 MRS_FIELD(ID_AA64MMFR3, S1PIE, false, MRS_LOWER, 0, id_aa64mmfr3_s1pie),
1609 MRS_FIELD(ID_AA64MMFR3, SCTLRX, false, MRS_LOWER, 0,
1610 id_aa64mmfr3_sctlrx),
1611 MRS_FIELD(ID_AA64MMFR3, TCRX, false, MRS_LOWER, 0, id_aa64mmfr3_tcrx),
1612 MRS_FIELD_END,
1613 };
1614
1615
1616 /* ID_AA64MMFR4_EL1 */
1617 static const struct mrs_field id_aa64mmfr4_fields[] = {
1618 MRS_FIELD_END,
1619 };
1620
1621
1622 /* ID_AA64PFR0_EL1 */
1623 static const struct mrs_field_value id_aa64pfr0_csv3[] = {
1624 MRS_FIELD_VALUE(ID_AA64PFR0_CSV3_NONE, ""),
1625 MRS_FIELD_VALUE(ID_AA64PFR0_CSV3_ISOLATED, "CSV3"),
1626 MRS_FIELD_VALUE_END,
1627 };
1628
1629 static const struct mrs_field_value id_aa64pfr0_csv2[] = {
1630 MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_NONE, ""),
1631 MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_ISOLATED, "CSV2"),
1632 MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_SCXTNUM, "CSV2_2"),
1633 MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_3, "CSV2_3"),
1634 MRS_FIELD_VALUE_END,
1635 };
1636
1637 static const struct mrs_field_value id_aa64pfr0_rme[] = {
1638 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, RME, NONE, IMPL),
1639 MRS_FIELD_VALUE_END,
1640 };
1641
1642 static const struct mrs_field_value id_aa64pfr0_dit[] = {
1643 MRS_FIELD_VALUE(ID_AA64PFR0_DIT_NONE, ""),
1644 MRS_FIELD_VALUE(ID_AA64PFR0_DIT_PSTATE, "PSTATE.DIT"),
1645 MRS_FIELD_VALUE_END,
1646 };
1647
1648 static const struct mrs_field_hwcap id_aa64pfr0_dit_caps[] = {
1649 MRS_HWCAP(1, HWCAP_DIT, ID_AA64PFR0_DIT_PSTATE),
1650 MRS_HWCAP_END
1651 };
1652
1653 static const struct mrs_field_value id_aa64pfr0_amu[] = {
1654 MRS_FIELD_VALUE(ID_AA64PFR0_AMU_NONE, ""),
1655 MRS_FIELD_VALUE(ID_AA64PFR0_AMU_V1, "AMUv1"),
1656 MRS_FIELD_VALUE(ID_AA64PFR0_AMU_V1_1, "AMUv1p1"),
1657 MRS_FIELD_VALUE_END,
1658 };
1659
1660 static const struct mrs_field_value id_aa64pfr0_mpam[] = {
1661 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, MPAM, NONE, IMPL),
1662 MRS_FIELD_VALUE_END,
1663 };
1664
1665 static const struct mrs_field_value id_aa64pfr0_sel2[] = {
1666 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, SEL2, NONE, IMPL),
1667 MRS_FIELD_VALUE_END,
1668 };
1669
1670 static const struct mrs_field_value id_aa64pfr0_sve[] = {
1671 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, SVE, NONE, IMPL),
1672 MRS_FIELD_VALUE_END,
1673 };
1674
1675 static const struct mrs_field_hwcap id_aa64pfr0_sve_caps[] = {
1676 MRS_HWCAP(1, HWCAP_SVE, ID_AA64PFR0_SVE_IMPL),
1677 MRS_HWCAP_END
1678 };
1679
1680 static const struct mrs_field_value id_aa64pfr0_ras[] = {
1681 MRS_FIELD_VALUE(ID_AA64PFR0_RAS_NONE, ""),
1682 MRS_FIELD_VALUE(ID_AA64PFR0_RAS_IMPL, "RAS"),
1683 MRS_FIELD_VALUE(ID_AA64PFR0_RAS_8_4, "RAS v8.4"),
1684 MRS_FIELD_VALUE(ID_AA64PFR0_RAS_8_9, "RAS v8.9"),
1685 MRS_FIELD_VALUE_END,
1686 };
1687
1688 static const struct mrs_field_value id_aa64pfr0_gic[] = {
1689 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, GIC, CPUIF_NONE, CPUIF_EN),
1690 MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_NONE, ""),
1691 MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_EN, "GIC"),
1692 MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_4_1, "GIC 4.1"),
1693 MRS_FIELD_VALUE_END,
1694 };
1695
1696 static const struct mrs_field_value id_aa64pfr0_advsimd[] = {
1697 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, AdvSIMD, NONE, IMPL),
1698 MRS_FIELD_VALUE(ID_AA64PFR0_AdvSIMD_HP, "AdvSIMD+HP"),
1699 MRS_FIELD_VALUE_END,
1700 };
1701
1702 static const struct mrs_field_hwcap id_aa64pfr0_advsimd_caps[] = {
1703 MRS_HWCAP(1, HWCAP_ASIMD, ID_AA64PFR0_AdvSIMD_IMPL),
1704 MRS_HWCAP(1, HWCAP_ASIMDHP, ID_AA64PFR0_AdvSIMD_HP),
1705 MRS_HWCAP_END
1706 };
1707
1708 static const struct mrs_field_value id_aa64pfr0_fp[] = {
1709 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, FP, NONE, IMPL),
1710 MRS_FIELD_VALUE(ID_AA64PFR0_FP_HP, "FP+HP"),
1711 MRS_FIELD_VALUE_END,
1712 };
1713
1714 static const struct mrs_field_hwcap id_aa64pfr0_fp_caps[] = {
1715 MRS_HWCAP(1, HWCAP_FP, ID_AA64PFR0_FP_IMPL),
1716 MRS_HWCAP(1, HWCAP_FPHP, ID_AA64PFR0_FP_HP),
1717 MRS_HWCAP_END
1718 };
1719
1720 static const struct mrs_field_value id_aa64pfr0_el3[] = {
1721 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, EL3, NONE, 64),
1722 MRS_FIELD_VALUE(ID_AA64PFR0_EL3_64_32, "EL3 32"),
1723 MRS_FIELD_VALUE_END,
1724 };
1725
1726 static const struct mrs_field_value id_aa64pfr0_el2[] = {
1727 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, EL2, NONE, 64),
1728 MRS_FIELD_VALUE(ID_AA64PFR0_EL2_64_32, "EL2 32"),
1729 MRS_FIELD_VALUE_END,
1730 };
1731
1732 static const struct mrs_field_value id_aa64pfr0_el1[] = {
1733 MRS_FIELD_VALUE(ID_AA64PFR0_EL1_64, "EL1"),
1734 MRS_FIELD_VALUE(ID_AA64PFR0_EL1_64_32, "EL1 32"),
1735 MRS_FIELD_VALUE_END,
1736 };
1737
1738 static const struct mrs_field_value id_aa64pfr0_el0[] = {
1739 MRS_FIELD_VALUE(ID_AA64PFR0_EL0_64, "EL0"),
1740 MRS_FIELD_VALUE(ID_AA64PFR0_EL0_64_32, "EL0 32"),
1741 MRS_FIELD_VALUE_END,
1742 };
1743
1744 static const struct mrs_field id_aa64pfr0_fields[] = {
1745 MRS_FIELD(ID_AA64PFR0, CSV3, false, MRS_LOWER, 0, id_aa64pfr0_csv3),
1746 MRS_FIELD(ID_AA64PFR0, CSV2, false, MRS_LOWER, 0, id_aa64pfr0_csv2),
1747 MRS_FIELD(ID_AA64PFR0, RME, false, MRS_LOWER, 0, id_aa64pfr0_rme),
1748 MRS_FIELD_HWCAP(ID_AA64PFR0, DIT, false, MRS_LOWER, MRS_USERSPACE,
1749 id_aa64pfr0_dit, id_aa64pfr0_dit_caps),
1750 MRS_FIELD(ID_AA64PFR0, AMU, false, MRS_LOWER, 0, id_aa64pfr0_amu),
1751 MRS_FIELD(ID_AA64PFR0, MPAM, false, MRS_LOWER, 0, id_aa64pfr0_mpam),
1752 MRS_FIELD(ID_AA64PFR0, SEL2, false, MRS_LOWER, 0, id_aa64pfr0_sel2),
1753 MRS_FIELD_HWCAP(ID_AA64PFR0, SVE, false, MRS_LOWER,
1754 MRS_FREEBSD, id_aa64pfr0_sve, id_aa64pfr0_sve_caps),
1755 MRS_FIELD(ID_AA64PFR0, RAS, false, MRS_LOWER, 0, id_aa64pfr0_ras),
1756 MRS_FIELD(ID_AA64PFR0, GIC, false, MRS_LOWER, 0, id_aa64pfr0_gic),
1757 MRS_FIELD_HWCAP(ID_AA64PFR0, AdvSIMD, true, MRS_LOWER, MRS_USERSPACE,
1758 id_aa64pfr0_advsimd, id_aa64pfr0_advsimd_caps),
1759 MRS_FIELD_HWCAP(ID_AA64PFR0, FP, true, MRS_LOWER, MRS_USERSPACE,
1760 id_aa64pfr0_fp, id_aa64pfr0_fp_caps),
1761 MRS_FIELD(ID_AA64PFR0, EL3, false, MRS_LOWER, 0, id_aa64pfr0_el3),
1762 MRS_FIELD(ID_AA64PFR0, EL2, false, MRS_LOWER, 0, id_aa64pfr0_el2),
1763 MRS_FIELD(ID_AA64PFR0, EL1, false, MRS_LOWER, MRS_USERSPACE,
1764 id_aa64pfr0_el1),
1765 MRS_FIELD(ID_AA64PFR0, EL0, false, MRS_LOWER, MRS_USERSPACE,
1766 id_aa64pfr0_el0),
1767 MRS_FIELD_END,
1768 };
1769
1770
1771 /* ID_AA64PFR1_EL1 */
1772 static const struct mrs_field_value id_aa64pfr1_pfar[] = {
1773 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR1, PFAR, NONE, IMPL),
1774 MRS_FIELD_VALUE_END,
1775 };
1776
1777 static const struct mrs_field_value id_aa64pfr1_df2[] = {
1778 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR1, DF2, NONE, IMPL),
1779 MRS_FIELD_VALUE_END,
1780 };
1781
1782 static const struct mrs_field_value id_aa64pfr1_mtex[] = {
1783 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR1, MTEX, NONE, IMPL),
1784 MRS_FIELD_VALUE_END,
1785 };
1786
1787 static const struct mrs_field_value id_aa64pfr1_the[] = {
1788 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR1, THE, NONE, IMPL),
1789 MRS_FIELD_VALUE_END,
1790 };
1791
1792 static const struct mrs_field_value id_aa64pfr1_mtefrac[] = {
1793 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR1, MTE_frac, NONE, IMPL),
1794 MRS_FIELD_VALUE_END,
1795 };
1796
1797 static const struct mrs_field_value id_aa64pfr1_nmi[] = {
1798 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR1, NMI, NONE, IMPL),
1799 MRS_FIELD_VALUE_END,
1800 };
1801
1802 static const struct mrs_field_value id_aa64pfr1_csv2_frac[] = {
1803 MRS_FIELD_VALUE(ID_AA64PFR1_CSV2_frac_p0, ""),
1804 MRS_FIELD_VALUE(ID_AA64PFR1_CSV2_frac_p1, "CSV2 p1"),
1805 MRS_FIELD_VALUE(ID_AA64PFR1_CSV2_frac_p2, "CSV2 p2"),
1806 MRS_FIELD_VALUE_END,
1807 };
1808
1809 static const struct mrs_field_value id_aa64pfr1_rndr_trap[] = {
1810 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR1, RNDR_trap, NONE, IMPL),
1811 MRS_FIELD_VALUE_END,
1812 };
1813
1814 static const struct mrs_field_value id_aa64pfr1_sme[] = {
1815 MRS_FIELD_VALUE(ID_AA64PFR1_SME_NONE, ""),
1816 MRS_FIELD_VALUE(ID_AA64PFR1_SME_SME, "SME"),
1817 MRS_FIELD_VALUE(ID_AA64PFR1_SME_SME2, "SME2"),
1818 MRS_FIELD_VALUE_END,
1819 };
1820
1821 static const struct mrs_field_value id_aa64pfr1_mpam_frac[] = {
1822 MRS_FIELD_VALUE(ID_AA64PFR1_MPAM_frac_p0, ""),
1823 MRS_FIELD_VALUE(ID_AA64PFR1_MPAM_frac_p1, "MPAM p1"),
1824 MRS_FIELD_VALUE_END,
1825 };
1826
1827 static const struct mrs_field_value id_aa64pfr1_ras_frac[] = {
1828 MRS_FIELD_VALUE(ID_AA64PFR1_RAS_frac_p0, ""),
1829 MRS_FIELD_VALUE(ID_AA64PFR1_RAS_frac_p1, "RAS p1"),
1830 MRS_FIELD_VALUE_END,
1831 };
1832
1833 static const struct mrs_field_value id_aa64pfr1_mte[] = {
1834 MRS_FIELD_VALUE(ID_AA64PFR1_MTE_NONE, ""),
1835 MRS_FIELD_VALUE(ID_AA64PFR1_MTE_MTE, "MTE"),
1836 MRS_FIELD_VALUE(ID_AA64PFR1_MTE_MTE2, "MTE2"),
1837 MRS_FIELD_VALUE(ID_AA64PFR1_MTE_MTE3, "MTE3"),
1838 MRS_FIELD_VALUE_END,
1839 };
1840
1841 static const struct mrs_field_value id_aa64pfr1_ssbs[] = {
1842 MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_NONE, ""),
1843 MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_PSTATE, "PSTATE.SSBS"),
1844 MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_PSTATE_MSR, "PSTATE.SSBS MSR"),
1845 MRS_FIELD_VALUE_END,
1846 };
1847
1848 static const struct mrs_field_hwcap id_aa64pfr1_ssbs_caps[] = {
1849 MRS_HWCAP(1, HWCAP_SSBS, ID_AA64PFR1_SSBS_PSTATE),
1850 MRS_HWCAP_END
1851 };
1852
1853 static const struct mrs_field_value id_aa64pfr1_bt[] = {
1854 MRS_FIELD_VALUE(ID_AA64PFR1_BT_NONE, ""),
1855 MRS_FIELD_VALUE(ID_AA64PFR1_BT_IMPL, "BTI"),
1856 MRS_FIELD_VALUE_END,
1857 };
1858
1859 static const struct mrs_field_hwcap id_aa64pfr1_bt_caps[] = {
1860 MRS_HWCAP(2, HWCAP2_BTI, ID_AA64PFR1_BT_IMPL),
1861 MRS_HWCAP_END
1862 };
1863
1864 static const struct mrs_field id_aa64pfr1_fields[] = {
1865 MRS_FIELD(ID_AA64PFR1, PFAR, false, MRS_LOWER, 0, id_aa64pfr1_pfar),
1866 MRS_FIELD(ID_AA64PFR1, DF2, false, MRS_LOWER, 0, id_aa64pfr1_df2),
1867 MRS_FIELD(ID_AA64PFR1, MTEX, false, MRS_LOWER, 0, id_aa64pfr1_mtex),
1868 MRS_FIELD(ID_AA64PFR1, THE, false, MRS_LOWER, 0, id_aa64pfr1_the),
1869 MRS_FIELD(ID_AA64PFR1, MTE_frac, true, MRS_LOWER, 0,
1870 id_aa64pfr1_mtefrac),
1871 MRS_FIELD(ID_AA64PFR1, NMI, false, MRS_LOWER, 0, id_aa64pfr1_nmi),
1872 MRS_FIELD(ID_AA64PFR1, CSV2_frac, false, MRS_LOWER, 0,
1873 id_aa64pfr1_csv2_frac),
1874 MRS_FIELD(ID_AA64PFR1, RNDR_trap, false, MRS_LOWER, 0,
1875 id_aa64pfr1_rndr_trap),
1876 MRS_FIELD(ID_AA64PFR1, SME, false, MRS_LOWER, 0, id_aa64pfr1_sme),
1877 MRS_FIELD(ID_AA64PFR1, MPAM_frac, false, MRS_LOWER, 0,
1878 id_aa64pfr1_mpam_frac),
1879 MRS_FIELD(ID_AA64PFR1, RAS_frac, false, MRS_LOWER, 0,
1880 id_aa64pfr1_ras_frac),
1881 MRS_FIELD(ID_AA64PFR1, MTE, false, MRS_LOWER, 0, id_aa64pfr1_mte),
1882 MRS_FIELD_HWCAP(ID_AA64PFR1, SSBS, false, MRS_LOWER, MRS_USERSPACE,
1883 id_aa64pfr1_ssbs, id_aa64pfr1_ssbs_caps),
1884 MRS_FIELD_HWCAP(ID_AA64PFR1, BT, false, MRS_LOWER,
1885 MRS_FREEBSD, id_aa64pfr1_bt, id_aa64pfr1_bt_caps),
1886 MRS_FIELD_END,
1887 };
1888
1889
1890 /* ID_AA64PFR2_EL1 */
1891 static const struct mrs_field id_aa64pfr2_fields[] = {
1892 MRS_FIELD_END,
1893 };
1894
1895
1896 /* ID_AA64ZFR0_EL1 */
1897 static const struct mrs_field_value id_aa64zfr0_f64mm[] = {
1898 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, F64MM, NONE, IMPL),
1899 MRS_FIELD_VALUE_END,
1900 };
1901
1902 static const struct mrs_field_hwcap id_aa64zfr0_f64mm_caps[] = {
1903 MRS_HWCAP(2, HWCAP2_SVEF64MM, ID_AA64ZFR0_F64MM_IMPL),
1904 MRS_HWCAP_END,
1905 };
1906
1907 static const struct mrs_field_value id_aa64zfr0_f32mm[] = {
1908 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, F32MM, NONE, IMPL),
1909 MRS_FIELD_VALUE_END,
1910 };
1911
1912 static const struct mrs_field_hwcap id_aa64zfr0_f32mm_caps[] = {
1913 MRS_HWCAP(2, HWCAP2_SVEF32MM, ID_AA64ZFR0_F32MM_IMPL),
1914 MRS_HWCAP_END,
1915 };
1916
1917 static const struct mrs_field_value id_aa64zfr0_i8mm[] = {
1918 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, I8MM, NONE, IMPL),
1919 MRS_FIELD_VALUE_END,
1920 };
1921
1922 static const struct mrs_field_hwcap id_aa64zfr0_i8mm_caps[] = {
1923 MRS_HWCAP(2, HWCAP2_SVEI8MM, ID_AA64ZFR0_I8MM_IMPL),
1924 MRS_HWCAP_END,
1925 };
1926
1927 static const struct mrs_field_value id_aa64zfr0_sm4[] = {
1928 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, SM4, NONE, IMPL),
1929 MRS_FIELD_VALUE_END,
1930 };
1931
1932 static const struct mrs_field_hwcap id_aa64zfr0_sm4_caps[] = {
1933 MRS_HWCAP(2, HWCAP2_SVESM4, ID_AA64ZFR0_SM4_IMPL),
1934 MRS_HWCAP_END,
1935 };
1936
1937 static const struct mrs_field_value id_aa64zfr0_sha3[] = {
1938 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, SHA3, NONE, IMPL),
1939 MRS_FIELD_VALUE_END,
1940 };
1941
1942 static const struct mrs_field_hwcap id_aa64zfr0_sha3_caps[] = {
1943 MRS_HWCAP(2, HWCAP2_SVESHA3, ID_AA64ZFR0_SHA3_IMPL),
1944 MRS_HWCAP_END,
1945 };
1946
1947 static const struct mrs_field_value id_aa64zfr0_bf16[] = {
1948 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, BF16, NONE, BASE),
1949 MRS_FIELD_VALUE(ID_AA64ZFR0_BF16_EBF, "BF16+EBF"),
1950 MRS_FIELD_VALUE_END,
1951 };
1952
1953 static const struct mrs_field_hwcap id_aa64zfr0_bf16_caps[] = {
1954 MRS_HWCAP(2, HWCAP2_SVEBF16, ID_AA64ZFR0_BF16_BASE),
1955 MRS_HWCAP(2, HWCAP2_SVE_EBF16, ID_AA64ZFR0_BF16_EBF),
1956 MRS_HWCAP_END,
1957 };
1958
1959 static const struct mrs_field_value id_aa64zfr0_bitperm[] = {
1960 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, BitPerm, NONE, IMPL),
1961 MRS_FIELD_VALUE_END,
1962 };
1963
1964 static const struct mrs_field_hwcap id_aa64zfr0_bitperm_caps[] = {
1965 MRS_HWCAP(2, HWCAP2_SVEBITPERM, ID_AA64ZFR0_BitPerm_IMPL),
1966 MRS_HWCAP_END,
1967 };
1968
1969 static const struct mrs_field_value id_aa64zfr0_aes[] = {
1970 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, AES, NONE, BASE),
1971 MRS_FIELD_VALUE(ID_AA64ZFR0_AES_PMULL, "AES+PMULL"),
1972 MRS_FIELD_VALUE_END,
1973 };
1974
1975 static const struct mrs_field_hwcap id_aa64zfr0_aes_caps[] = {
1976 MRS_HWCAP(2, HWCAP2_SVEAES, ID_AA64ZFR0_AES_BASE),
1977 MRS_HWCAP(2, HWCAP2_SVEPMULL, ID_AA64ZFR0_AES_PMULL),
1978 MRS_HWCAP_END,
1979 };
1980
1981 static const struct mrs_field_value id_aa64zfr0_svever[] = {
1982 MRS_FIELD_VALUE(ID_AA64ZFR0_SVEver_SVE1, "SVE1"),
1983 MRS_FIELD_VALUE(ID_AA64ZFR0_SVEver_SVE2, "SVE2"),
1984 MRS_FIELD_VALUE(ID_AA64ZFR0_SVEver_SVE2P1, "SVE2P1"),
1985 MRS_FIELD_VALUE_END,
1986 };
1987
1988 static const struct mrs_field_hwcap id_aa64zfr0_svever_caps[] = {
1989 MRS_HWCAP(2, HWCAP2_SVE2, ID_AA64ZFR0_SVEver_SVE2),
1990 MRS_HWCAP(2, HWCAP2_SVE2P1, ID_AA64ZFR0_SVEver_SVE2P1),
1991 MRS_HWCAP_END,
1992 };
1993
1994 static const struct mrs_field id_aa64zfr0_fields[] = {
1995 MRS_FIELD_HWCAP(ID_AA64ZFR0, F64MM, false, MRS_LOWER, MRS_USERSPACE,
1996 id_aa64zfr0_f64mm, id_aa64zfr0_f64mm_caps),
1997 MRS_FIELD_HWCAP(ID_AA64ZFR0, F32MM, false, MRS_LOWER, MRS_USERSPACE,
1998 id_aa64zfr0_f32mm, id_aa64zfr0_f32mm_caps),
1999 MRS_FIELD_HWCAP(ID_AA64ZFR0, I8MM, false, MRS_LOWER, MRS_USERSPACE,
2000 id_aa64zfr0_i8mm, id_aa64zfr0_i8mm_caps),
2001 MRS_FIELD_HWCAP(ID_AA64ZFR0, SM4, false, MRS_LOWER, MRS_USERSPACE,
2002 id_aa64zfr0_sm4, id_aa64zfr0_sm4_caps),
2003 MRS_FIELD_HWCAP(ID_AA64ZFR0, SHA3, false, MRS_LOWER, MRS_USERSPACE,
2004 id_aa64zfr0_sha3, id_aa64zfr0_sha3_caps),
2005 MRS_FIELD_HWCAP(ID_AA64ZFR0, BF16, false, MRS_LOWER, MRS_USERSPACE,
2006 id_aa64zfr0_bf16, id_aa64zfr0_bf16_caps),
2007 MRS_FIELD_HWCAP(ID_AA64ZFR0, BitPerm, false, MRS_LOWER, MRS_USERSPACE,
2008 id_aa64zfr0_bitperm, id_aa64zfr0_bitperm_caps),
2009 MRS_FIELD_HWCAP(ID_AA64ZFR0, AES, false, MRS_LOWER, MRS_USERSPACE,
2010 id_aa64zfr0_aes, id_aa64zfr0_aes_caps),
2011 MRS_FIELD_HWCAP(ID_AA64ZFR0, SVEver, false, MRS_LOWER, MRS_USERSPACE,
2012 id_aa64zfr0_svever, id_aa64zfr0_svever_caps),
2013 MRS_FIELD_END,
2014 };
2015
2016
2017 #ifdef COMPAT_FREEBSD32
2018 /* ID_ISAR5_EL1 */
2019 static const struct mrs_field_value id_isar5_vcma[] = {
2020 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, VCMA, NONE, IMPL),
2021 MRS_FIELD_VALUE_END,
2022 };
2023
2024 static const struct mrs_field_value id_isar5_rdm[] = {
2025 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, RDM, NONE, IMPL),
2026 MRS_FIELD_VALUE_END,
2027 };
2028
2029 static const struct mrs_field_value id_isar5_crc32[] = {
2030 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, CRC32, NONE, IMPL),
2031 MRS_FIELD_VALUE_END,
2032 };
2033
2034 static const struct mrs_field_hwcap id_isar5_crc32_caps[] = {
2035 MRS_HWCAP(2, HWCAP32_2_CRC32, ID_ISAR5_CRC32_IMPL),
2036 MRS_HWCAP_END
2037 };
2038
2039 static const struct mrs_field_value id_isar5_sha2[] = {
2040 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SHA2, NONE, IMPL),
2041 MRS_FIELD_VALUE_END,
2042 };
2043
2044 static const struct mrs_field_hwcap id_isar5_sha2_caps[] = {
2045 MRS_HWCAP(2, HWCAP32_2_SHA2, ID_ISAR5_SHA2_IMPL),
2046 MRS_HWCAP_END
2047 };
2048
2049 static const struct mrs_field_value id_isar5_sha1[] = {
2050 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SHA1, NONE, IMPL),
2051 MRS_FIELD_VALUE_END,
2052 };
2053
2054 static const struct mrs_field_hwcap id_isar5_sha1_caps[] = {
2055 MRS_HWCAP(2, HWCAP32_2_SHA1, ID_ISAR5_SHA1_IMPL),
2056 MRS_HWCAP_END
2057 };
2058
2059 static const struct mrs_field_value id_isar5_aes[] = {
2060 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, AES, NONE, BASE),
2061 MRS_FIELD_VALUE(ID_ISAR5_AES_VMULL, "AES+VMULL"),
2062 MRS_FIELD_VALUE_END,
2063 };
2064
2065 static const struct mrs_field_hwcap id_isar5_aes_caps[] = {
2066 MRS_HWCAP(2, HWCAP32_2_AES, ID_ISAR5_AES_BASE),
2067 MRS_HWCAP(2, HWCAP32_2_PMULL, ID_ISAR5_AES_VMULL),
2068 MRS_HWCAP_END
2069 };
2070
2071 static const struct mrs_field_value id_isar5_sevl[] = {
2072 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SEVL, NOP, IMPL),
2073 MRS_FIELD_VALUE_END,
2074 };
2075
2076 static const struct mrs_field id_isar5_fields[] = {
2077 MRS_FIELD(ID_ISAR5, VCMA, false, MRS_LOWER,MRS_USERSPACE,
2078 id_isar5_vcma),
2079 MRS_FIELD(ID_ISAR5, RDM, false, MRS_LOWER, MRS_USERSPACE, id_isar5_rdm),
2080 MRS_FIELD_HWCAP(ID_ISAR5, CRC32, false, MRS_LOWER, MRS_USERSPACE,
2081 id_isar5_crc32, id_isar5_crc32_caps),
2082 MRS_FIELD_HWCAP(ID_ISAR5, SHA2, false, MRS_LOWER, MRS_USERSPACE,
2083 id_isar5_sha2, id_isar5_sha2_caps),
2084 MRS_FIELD_HWCAP(ID_ISAR5, SHA1, false, MRS_LOWER, MRS_USERSPACE,
2085 id_isar5_sha1, id_isar5_sha1_caps),
2086 MRS_FIELD_HWCAP(ID_ISAR5, AES, false, MRS_LOWER, MRS_USERSPACE,
2087 id_isar5_aes, id_isar5_aes_caps),
2088 MRS_FIELD(ID_ISAR5, SEVL, false, MRS_LOWER, MRS_USERSPACE,
2089 id_isar5_sevl),
2090 MRS_FIELD_END,
2091 };
2092
2093 /* MVFR0 */
2094 static const struct mrs_field_value mvfr0_fpround[] = {
2095 MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPRound, NONE, IMPL),
2096 MRS_FIELD_VALUE_END,
2097 };
2098
2099 static const struct mrs_field_value mvfr0_fpsqrt[] = {
2100 MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPSqrt, NONE, IMPL),
2101 MRS_FIELD_VALUE_END,
2102 };
2103
2104 static const struct mrs_field_value mvfr0_fpdivide[] = {
2105 MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPDivide, NONE, IMPL),
2106 MRS_FIELD_VALUE_END,
2107 };
2108
2109 static const struct mrs_field_value mvfr0_fptrap[] = {
2110 MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPTrap, NONE, IMPL),
2111 MRS_FIELD_VALUE_END,
2112 };
2113
2114 static const struct mrs_field_value mvfr0_fpdp[] = {
2115 MRS_FIELD_VALUE(MVFR0_FPDP_NONE, ""),
2116 MRS_FIELD_VALUE(MVFR0_FPDP_VFP_v2, "DP VFPv2"),
2117 MRS_FIELD_VALUE(MVFR0_FPDP_VFP_v3_v4, "DP VFPv3+v4"),
2118 MRS_FIELD_VALUE_END,
2119 };
2120
2121 static const struct mrs_field_hwcap mvfr0_fpdp_caps[] = {
2122 MRS_HWCAP(1, HWCAP32_VFP, MVFR0_FPDP_VFP_v2),
2123 MRS_HWCAP(1, HWCAP32_VFPv3, MVFR0_FPDP_VFP_v3_v4),
2124 MRS_HWCAP_END
2125 };
2126
2127 static const struct mrs_field_value mvfr0_fpsp[] = {
2128 MRS_FIELD_VALUE(MVFR0_FPSP_NONE, ""),
2129 MRS_FIELD_VALUE(MVFR0_FPSP_VFP_v2, "SP VFPv2"),
2130 MRS_FIELD_VALUE(MVFR0_FPSP_VFP_v3_v4, "SP VFPv3+v4"),
2131 MRS_FIELD_VALUE_END,
2132 };
2133
2134 static const struct mrs_field_value mvfr0_simdreg[] = {
2135 MRS_FIELD_VALUE(MVFR0_SIMDReg_NONE, ""),
2136 MRS_FIELD_VALUE(MVFR0_SIMDReg_FP, "FP 16x64"),
2137 MRS_FIELD_VALUE(MVFR0_SIMDReg_AdvSIMD, "AdvSIMD"),
2138 MRS_FIELD_VALUE_END,
2139 };
2140
2141 static const struct mrs_field mvfr0_fields[] = {
2142 MRS_FIELD(MVFR0, FPRound, false, MRS_LOWER, MRS_USERSPACE,
2143 mvfr0_fpround),
2144 MRS_FIELD(MVFR0, FPSqrt, false, MRS_LOWER, MRS_USERSPACE,
2145 mvfr0_fpsqrt),
2146 MRS_FIELD(MVFR0, FPDivide, false, MRS_LOWER, MRS_USERSPACE,
2147 mvfr0_fpdivide),
2148 MRS_FIELD(MVFR0, FPTrap, false, MRS_LOWER, MRS_USERSPACE,
2149 mvfr0_fptrap),
2150 MRS_FIELD_HWCAP(MVFR0, FPDP, false, MRS_LOWER, MRS_USERSPACE,
2151 mvfr0_fpdp, mvfr0_fpdp_caps),
2152 MRS_FIELD(MVFR0, FPSP, false, MRS_LOWER, MRS_USERSPACE, mvfr0_fpsp),
2153 MRS_FIELD(MVFR0, SIMDReg, false, MRS_LOWER, MRS_USERSPACE,
2154 mvfr0_simdreg),
2155 MRS_FIELD_END,
2156 };
2157
2158 /* MVFR1 */
2159 static const struct mrs_field_value mvfr1_simdfmac[] = {
2160 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDFMAC, NONE, IMPL),
2161 MRS_FIELD_VALUE_END,
2162 };
2163
2164 static const struct mrs_field_hwcap mvfr1_simdfmac_caps[] = {
2165 MRS_HWCAP(1, HWCAP32_VFPv4, MVFR1_SIMDFMAC_IMPL),
2166 MRS_HWCAP_END
2167 };
2168
2169 static const struct mrs_field_value mvfr1_fphp[] = {
2170 MRS_FIELD_VALUE(MVFR1_FPHP_NONE, ""),
2171 MRS_FIELD_VALUE(MVFR1_FPHP_CONV_SP, "FPHP SP Conv"),
2172 MRS_FIELD_VALUE(MVFR1_FPHP_CONV_DP, "FPHP DP Conv"),
2173 MRS_FIELD_VALUE(MVFR1_FPHP_ARITH, "FPHP Arith"),
2174 MRS_FIELD_VALUE_END,
2175 };
2176
2177 static const struct mrs_field_value mvfr1_simdhp[] = {
2178 MRS_FIELD_VALUE(MVFR1_SIMDHP_NONE, ""),
2179 MRS_FIELD_VALUE(MVFR1_SIMDHP_CONV_SP, "SIMDHP SP Conv"),
2180 MRS_FIELD_VALUE(MVFR1_SIMDHP_ARITH, "SIMDHP Arith"),
2181 MRS_FIELD_VALUE_END,
2182 };
2183
2184 static const struct mrs_field_value mvfr1_simdsp[] = {
2185 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDSP, NONE, IMPL),
2186 MRS_FIELD_VALUE_END,
2187 };
2188
2189 static const struct mrs_field_value mvfr1_simdint[] = {
2190 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDInt, NONE, IMPL),
2191 MRS_FIELD_VALUE_END,
2192 };
2193
2194 static const struct mrs_field_value mvfr1_simdls[] = {
2195 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDLS, NONE, IMPL),
2196 MRS_FIELD_VALUE_END,
2197 };
2198
2199 static const struct mrs_field_hwcap mvfr1_simdls_caps[] = {
2200 MRS_HWCAP(1, HWCAP32_VFPv4, MVFR1_SIMDFMAC_IMPL),
2201 MRS_HWCAP_END
2202 };
2203
2204 static const struct mrs_field_value mvfr1_fpdnan[] = {
2205 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, FPDNaN, NONE, IMPL),
2206 MRS_FIELD_VALUE_END,
2207 };
2208
2209 static const struct mrs_field_value mvfr1_fpftz[] = {
2210 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, FPFtZ, NONE, IMPL),
2211 MRS_FIELD_VALUE_END,
2212 };
2213
2214 static const struct mrs_field mvfr1_fields[] = {
2215 MRS_FIELD_HWCAP(MVFR1, SIMDFMAC, false, MRS_LOWER, MRS_USERSPACE,
2216 mvfr1_simdfmac, mvfr1_simdfmac_caps),
2217 MRS_FIELD(MVFR1, FPHP, false, MRS_LOWER, MRS_USERSPACE, mvfr1_fphp),
2218 MRS_FIELD(MVFR1, SIMDHP, false, MRS_LOWER, MRS_USERSPACE, mvfr1_simdhp),
2219 MRS_FIELD(MVFR1, SIMDSP, false, MRS_LOWER, MRS_USERSPACE, mvfr1_simdsp),
2220 MRS_FIELD(MVFR1, SIMDInt, false, MRS_LOWER, MRS_USERSPACE,
2221 mvfr1_simdint),
2222 MRS_FIELD_HWCAP(MVFR1, SIMDLS, false, MRS_LOWER, MRS_USERSPACE,
2223 mvfr1_simdls, mvfr1_simdls_caps),
2224 MRS_FIELD(MVFR1, FPDNaN, false, MRS_LOWER, MRS_USERSPACE,
2225 mvfr1_fpdnan),
2226 MRS_FIELD(MVFR1, FPFtZ, false, MRS_LOWER, MRS_USERSPACE,
2227 mvfr1_fpftz),
2228 MRS_FIELD_END,
2229 };
2230 #endif /* COMPAT_FREEBSD32 */
2231
2232 struct mrs_user_reg {
2233 u_int iss;
2234 bool is64bit;
2235 size_t offset;
2236 const struct mrs_field *fields;
2237 };
2238
2239 #define USER_REG(name, field_name, _is64bit) \
2240 { \
2241 .iss = name##_ISS, \
2242 .offset = __offsetof(struct cpu_desc, field_name), \
2243 .fields = field_name##_fields, \
2244 .is64bit = _is64bit, \
2245 }
2246 static const struct mrs_user_reg user_regs[] = {
2247 USER_REG(ID_AA64AFR0_EL1, id_aa64afr0, true),
2248 USER_REG(ID_AA64AFR1_EL1, id_aa64afr1, true),
2249
2250 USER_REG(ID_AA64DFR0_EL1, id_aa64dfr0, true),
2251 USER_REG(ID_AA64DFR1_EL1, id_aa64dfr1, true),
2252
2253 USER_REG(ID_AA64ISAR0_EL1, id_aa64isar0, true),
2254 USER_REG(ID_AA64ISAR1_EL1, id_aa64isar1, true),
2255 USER_REG(ID_AA64ISAR2_EL1, id_aa64isar2, true),
2256
2257 USER_REG(ID_AA64MMFR0_EL1, id_aa64mmfr0, true),
2258 USER_REG(ID_AA64MMFR1_EL1, id_aa64mmfr1, true),
2259 USER_REG(ID_AA64MMFR2_EL1, id_aa64mmfr2, true),
2260 USER_REG(ID_AA64MMFR3_EL1, id_aa64mmfr3, true),
2261 USER_REG(ID_AA64MMFR4_EL1, id_aa64mmfr4, true),
2262
2263 USER_REG(ID_AA64PFR0_EL1, id_aa64pfr0, true),
2264 USER_REG(ID_AA64PFR1_EL1, id_aa64pfr1, true),
2265 USER_REG(ID_AA64PFR2_EL1, id_aa64pfr2, true),
2266
2267 USER_REG(ID_AA64ZFR0_EL1, id_aa64zfr0, true),
2268
2269 USER_REG(CTR_EL0, ctr, true),
2270
2271 #ifdef COMPAT_FREEBSD32
2272 USER_REG(ID_ISAR5_EL1, id_isar5, false),
2273
2274 USER_REG(MVFR0_EL1, mvfr0, false),
2275 USER_REG(MVFR1_EL1, mvfr1, false),
2276 #endif /* COMPAT_FREEBSD32 */
2277 };
2278
2279 #define CPU_DESC_FIELD(desc, idx) \
2280 *(uint64_t *)((char *)&(desc) + user_regs[(idx)].offset)
2281
2282 static bool
user_ctr_has_neoverse_n1_1542419(uint32_t midr,uint64_t ctr)2283 user_ctr_has_neoverse_n1_1542419(uint32_t midr, uint64_t ctr)
2284 {
2285 /*
2286 * Neoverse-N1 erratum 1542419
2287 * Present in r3p0 - r4p0
2288 * Fixed in r4p1
2289 */
2290 return (midr_check_var_part_range(midr, CPU_IMPL_ARM,
2291 CPU_PART_NEOVERSE_N1, 3, 0, 4, 0) && CTR_DIC_VAL(ctr) != 0);
2292 }
2293
2294 static cpu_feat_en
user_ctr_check(const struct cpu_feat * feat __unused,u_int midr)2295 user_ctr_check(const struct cpu_feat *feat __unused, u_int midr)
2296 {
2297 if (emulate_ctr)
2298 return (FEAT_DEFAULT_ENABLE);
2299
2300 if (user_ctr_has_neoverse_n1_1542419(midr, READ_SPECIALREG(ctr_el0)))
2301 return (FEAT_DEFAULT_ENABLE);
2302
2303 return (FEAT_ALWAYS_DISABLE);
2304 }
2305
2306 static bool
user_ctr_has_errata(const struct cpu_feat * feat __unused,u_int midr,u_int ** errata_list,u_int * errata_count)2307 user_ctr_has_errata(const struct cpu_feat *feat __unused, u_int midr,
2308 u_int **errata_list, u_int *errata_count)
2309 {
2310 if (user_ctr_has_neoverse_n1_1542419(midr, READ_SPECIALREG(ctr_el0))) {
2311 static u_int errata_id = 1542419;
2312
2313 *errata_list = &errata_id;
2314 *errata_count = 1;
2315 return (true);
2316 }
2317
2318 return (false);
2319 }
2320
2321 static bool
user_ctr_enable(const struct cpu_feat * feat __unused,cpu_feat_errata errata_status,u_int * errata_list,u_int errata_count)2322 user_ctr_enable(const struct cpu_feat *feat __unused,
2323 cpu_feat_errata errata_status, u_int *errata_list, u_int errata_count)
2324 {
2325 MPASS(emulate_ctr || errata_status != ERRATA_NONE);
2326
2327 /*
2328 * The Errata Management Firmware Interface may incorrectly mark
2329 * this as firmware mitigated. We should ignore that as there is
2330 * a kernel component to the mitigation.
2331 */
2332 if (errata_status != ERRATA_NONE && PCPU_GET(cpuid) == 0 &&
2333 cpu_feat_has_erratum(errata_list, errata_count, 1542419)) {
2334 /* Clear fields we will change */
2335 user_cpu_desc.ctr &= ~(CTR_DIC_MASK | CTR_ILINE_WIDTH);
2336
2337 /*
2338 * Set DIC to none so userspace will execute an 'ic ivau'
2339 * instruction that can be trapped by EL3.
2340 */
2341 user_cpu_desc.ctr |= CTR_DIC_NONE;
2342 /*
2343 * Set the i-cache line size to be page size to reduce the
2344 * number of times userspace needs to execute the 'ic ivau'
2345 * instruction. The ctr_el0.IminLine is log2 the number of
2346 * 4-byte words the instruction covers. As PAGE_SHIFT is log2
2347 * of the number of bytes in a page we need to subtract 2.
2348 */
2349 user_cpu_desc.ctr |= (PAGE_SHIFT - 2) << CTR_ILINE_SHIFT;
2350
2351 l_user_cpu_desc.ctr = user_cpu_desc.ctr;
2352 }
2353
2354 WRITE_SPECIALREG(sctlr_el1,
2355 READ_SPECIALREG(sctlr_el1) & ~SCTLR_UCT);
2356 isb();
2357
2358 return (true);
2359 }
2360
2361 CPU_FEAT(trap_ctr, "Trap CTR_EL0",
2362 user_ctr_check, user_ctr_has_errata, user_ctr_enable, NULL,
2363 CPU_FEAT_AFTER_DEV | CPU_FEAT_PER_CPU);
2364
2365 static bool
user_ctr_handler(uint64_t esr,struct trapframe * frame)2366 user_ctr_handler(uint64_t esr, struct trapframe *frame)
2367 {
2368 uint64_t value;
2369 int reg;
2370
2371 if (ESR_ELx_EXCEPTION(esr) != EXCP_MSR)
2372 return (false);
2373
2374 /* Only support reading from ctr_el0 */
2375 if ((esr & ISS_MSR_DIR) == 0)
2376 return (false);
2377
2378 /* Check if this is the ctr_el0 register */
2379 if ((esr & ISS_MSR_REG_MASK) != CTR_EL0_ISS)
2380 return (false);
2381
2382 if (SV_CURPROC_ABI() == SV_ABI_FREEBSD)
2383 value = user_cpu_desc.ctr;
2384 else
2385 value = l_user_cpu_desc.ctr;
2386 /*
2387 * We will handle this instruction, move to the next so we
2388 * don't trap here again.
2389 */
2390 frame->tf_elr += INSN_SIZE;
2391
2392 reg = ISS_MSR_Rt(esr);
2393 /* If reg is 31 then write to xzr, i.e. do nothing */
2394 if (reg == 31)
2395 return (true);
2396
2397 if (reg < nitems(frame->tf_x))
2398 frame->tf_x[reg] = value;
2399 else if (reg == 30)
2400 frame->tf_lr = value;
2401
2402 return (true);
2403 }
2404
2405 static bool
user_idreg_handler(uint64_t esr,struct trapframe * frame)2406 user_idreg_handler(uint64_t esr, struct trapframe *frame)
2407 {
2408 uint64_t value;
2409 int reg;
2410
2411 if (ESR_ELx_EXCEPTION(esr) != EXCP_MSR)
2412 return (false);
2413
2414 /* Only support reading from ID registers */
2415 if ((esr & ISS_MSR_DIR) == 0)
2416 return (false);
2417
2418 /*
2419 * This only handles the ID register space and a few registers that
2420 * are safe to pass through to userspace.
2421 *
2422 * These registers are all in the space op0 == 3, op1 == 0,
2423 * CRn == 0. We support the following CRm:
2424 * - CRm == 0: midr_el1, mpidr_el1, and revidr_el1.
2425 * - CRm in {4-7}: sanitized ID registers.
2426 *
2427 * Registers in the ID register space (CRm in {4-7}) are all
2428 * read-only and have either defined fields, or are read as
2429 * zero (RAZ). For these we return 0 for any unknown register.
2430 */
2431 if (ISS_MSR_OP0(esr) != 3 || ISS_MSR_OP1(esr) != 0 ||
2432 ISS_MSR_CRn(esr) != 0)
2433 return (false);
2434
2435 value = 0;
2436 if (ISS_MSR_CRm(esr) >= 4 && ISS_MSR_CRm(esr) <= 7) {
2437 for (int i = 0; i < nitems(user_regs); i++) {
2438 if (user_regs[i].iss == (esr & ISS_MSR_REG_MASK)) {
2439 if (SV_CURPROC_ABI() == SV_ABI_FREEBSD)
2440 value = CPU_DESC_FIELD(user_cpu_desc, i);
2441 else
2442 value = CPU_DESC_FIELD(l_user_cpu_desc, i);
2443 break;
2444 }
2445 }
2446 } else if (ISS_MSR_CRm(esr) == 0) {
2447 switch (ISS_MSR_OP2(esr)) {
2448 case 0:
2449 value = READ_SPECIALREG(midr_el1);
2450 break;
2451 case 5:
2452 value = READ_SPECIALREG(mpidr_el1);
2453 break;
2454 case 6:
2455 value = READ_SPECIALREG(revidr_el1);
2456 break;
2457 default:
2458 return (false);
2459 }
2460 } else {
2461 return (false);
2462 }
2463
2464 /*
2465 * We will handle this instruction, move to the next so we
2466 * don't trap here again.
2467 */
2468 frame->tf_elr += INSN_SIZE;
2469
2470 reg = ISS_MSR_Rt(esr);
2471 /* If reg is 31 then write to xzr, i.e. do nothing */
2472 if (reg == 31)
2473 return (1);
2474
2475 if (reg < nitems(frame->tf_x))
2476 frame->tf_x[reg] = value;
2477 else if (reg == 30)
2478 frame->tf_lr = value;
2479
2480 return (true);
2481 }
2482
2483 /*
2484 * Compares two field values that may be signed or unsigned.
2485 * Returns:
2486 * < 0 when a is less than b
2487 * = 0 when a equals b
2488 * > 0 when a is greater than b
2489 */
2490 static int
mrs_field_cmp(uint64_t a,uint64_t b,u_int shift,int width,bool sign)2491 mrs_field_cmp(uint64_t a, uint64_t b, u_int shift, int width, bool sign)
2492 {
2493 uint64_t mask;
2494
2495 KASSERT(width > 0 && width < 64, ("%s: Invalid width %d", __func__,
2496 width));
2497
2498 mask = (1ul << width) - 1;
2499 /* Move the field to the lower bits */
2500 a = (a >> shift) & mask;
2501 b = (b >> shift) & mask;
2502
2503 if (sign) {
2504 /*
2505 * The field is signed. Toggle the upper bit so the comparison
2506 * works on unsigned values as this makes positive numbers,
2507 * i.e. those with a 0 bit, larger than negative numbers,
2508 * i.e. those with a 1 bit, in an unsigned comparison.
2509 */
2510 a ^= 1ul << (width - 1);
2511 b ^= 1ul << (width - 1);
2512 }
2513
2514 return (a - b);
2515 }
2516
2517 bool
get_kernel_reg_iss(u_int iss,uint64_t * val)2518 get_kernel_reg_iss(u_int iss, uint64_t *val)
2519 {
2520 int i;
2521
2522 for (i = 0; i < nitems(user_regs); i++) {
2523 if (user_regs[i].iss == iss) {
2524 *val = CPU_DESC_FIELD(kern_cpu_desc, i);
2525 return (true);
2526 }
2527 }
2528
2529 return (false);
2530 }
2531
2532 /*
2533 * Fetch the specified register's value, ensuring that individual field values
2534 * do not exceed those in the mask.
2535 */
2536 bool
get_kernel_reg_iss_masked(u_int iss,uint64_t * valp,uint64_t mask)2537 get_kernel_reg_iss_masked(u_int iss, uint64_t *valp, uint64_t mask)
2538 {
2539 const struct mrs_field *fields;
2540 uint64_t val;
2541
2542 for (int i = 0; i < nitems(user_regs); i++) {
2543 if (user_regs[i].iss == iss) {
2544 val = CPU_DESC_FIELD(kern_cpu_desc, i);
2545 fields = user_regs[i].fields;
2546 for (int j = 0; fields[j].type != 0; j++) {
2547 mask = update_special_reg_field(mask,
2548 fields[j].type, val, fields[j].width,
2549 fields[j].shift, fields[j].sign);
2550 }
2551 *valp = mask;
2552 return (true);
2553 }
2554 }
2555
2556 return (false);
2557 }
2558
2559 bool
get_user_reg_iss(u_int iss,uint64_t * val,bool fbsd)2560 get_user_reg_iss(u_int iss, uint64_t *val, bool fbsd)
2561 {
2562 int i;
2563
2564 for (i = 0; i < nitems(user_regs); i++) {
2565 if (user_regs[i].iss == iss) {
2566 if (fbsd)
2567 *val = CPU_DESC_FIELD(user_cpu_desc, i);
2568 else
2569 *val = CPU_DESC_FIELD(l_user_cpu_desc, i);
2570 return (true);
2571 }
2572 }
2573
2574 return (false);
2575 }
2576
2577 static uint64_t
update_special_reg_field(uint64_t user_reg,u_int type,uint64_t value,u_int width,u_int shift,bool sign)2578 update_special_reg_field(uint64_t user_reg, u_int type, uint64_t value,
2579 u_int width, u_int shift, bool sign)
2580 {
2581 uint64_t cur, mask, new_val;
2582
2583 mask = ((1ul << width) - 1) << shift;
2584 cur = user_reg & mask;
2585 new_val = value & mask;
2586
2587 switch (type & MRS_TYPE_MASK) {
2588 case MRS_EXACT_IF_DIFFERENT:
2589 if (mrs_field_cmp(new_val, cur, shift, width, sign) == 0)
2590 break;
2591 /* FALLTHROUGH */
2592 case MRS_EXACT:
2593 cur = (uint64_t)MRS_SAFE_VAL(type) << shift;
2594 break;
2595 case MRS_LOWER:
2596 if (mrs_field_cmp(new_val, cur, shift, width, sign) < 0)
2597 cur = new_val;
2598 break;
2599 case MRS_HIGHER_OR_ZERO:
2600 if (cur == 0 || new_val == 0) {
2601 cur = 0;
2602 break;
2603 }
2604 /* FALLTHROUGH */
2605 case MRS_HIGHER:
2606 if (mrs_field_cmp(new_val, cur, shift, width, sign) > 0)
2607 cur = new_val;
2608 break;
2609 default:
2610 panic("Invalid field type: %d", type);
2611 }
2612
2613 user_reg &= ~mask;
2614 user_reg |= cur;
2615
2616 return (user_reg);
2617 }
2618
2619 static void
clear_set_special_reg_idx(int idx,uint64_t clear,uint64_t set)2620 clear_set_special_reg_idx(int idx, uint64_t clear, uint64_t set)
2621 {
2622 const struct mrs_field *fields;
2623 uint64_t k_old, k_new;
2624 uint64_t f_old, f_new;
2625 uint64_t l_old, l_new;
2626
2627 MPASS(idx < nitems(user_regs));
2628
2629 k_old = CPU_DESC_FIELD(kern_cpu_desc, idx);
2630 k_new = (k_old & ~clear) | set;
2631
2632 f_old = CPU_DESC_FIELD(user_cpu_desc, idx);
2633 f_new = (f_old & ~clear) | set;
2634
2635 l_old = CPU_DESC_FIELD(l_user_cpu_desc, idx);
2636 l_new = (l_old & ~clear) | set;
2637
2638 fields = user_regs[idx].fields;
2639 for (int j = 0; fields[j].type != 0; j++) {
2640 u_int type;
2641
2642 /* Update the FreeBSD userspace ID register view */
2643 type = ((fields[j].type & MRS_FREEBSD) != 0) ?
2644 fields[j].type :
2645 (MRS_EXACT | (fields[j].type & MRS_SAFE_MASK));
2646 f_new = update_special_reg_field(f_new,
2647 type, f_old, fields[j].width, fields[j].shift,
2648 fields[j].sign);
2649
2650 /* Update the Linux userspace ID register view */
2651 type = ((fields[j].type & MRS_LINUX) != 0) ?
2652 fields[j].type :
2653 (MRS_EXACT | (fields[j].type & MRS_SAFE_MASK));
2654 l_new = update_special_reg_field(l_new,
2655 type, l_old, fields[j].width, fields[j].shift,
2656 fields[j].sign);
2657
2658 /* Update the kernel ID register view */
2659 k_new = update_special_reg_field(k_new,
2660 fields[j].type, k_old, fields[j].width,
2661 fields[j].shift, fields[j].sign);
2662 }
2663
2664 CPU_DESC_FIELD(kern_cpu_desc, idx) = k_new;
2665 CPU_DESC_FIELD(user_cpu_desc, idx) = f_new;
2666 CPU_DESC_FIELD(l_user_cpu_desc, idx) = l_new;
2667 }
2668
2669 void
update_special_regs(u_int cpu)2670 update_special_regs(u_int cpu)
2671 {
2672 struct cpu_desc *desc;
2673 uint64_t value;
2674 int i;
2675
2676 if (cpu == 0) {
2677 /* Create a user visible cpu description with safe values */
2678 memset(&user_cpu_desc, 0, sizeof(user_cpu_desc));
2679 /* Safe values for these registers */
2680 user_cpu_desc.id_aa64pfr0 = ID_AA64PFR0_AdvSIMD_NONE |
2681 ID_AA64PFR0_FP_NONE | ID_AA64PFR0_EL1_64 |
2682 ID_AA64PFR0_EL0_64;
2683 user_cpu_desc.id_aa64dfr0 = ID_AA64DFR0_DebugVer_8;
2684 /* Create the Linux user visible cpu description */
2685 memcpy(&l_user_cpu_desc, &user_cpu_desc, sizeof(user_cpu_desc));
2686 }
2687
2688 desc = get_cpu_desc(cpu);
2689 for (i = 0; i < nitems(user_regs); i++) {
2690 value = CPU_DESC_FIELD(*desc, i);
2691 if (cpu == 0) {
2692 CPU_DESC_FIELD(kern_cpu_desc, i) = value;
2693 CPU_DESC_FIELD(user_cpu_desc, i) = value;
2694 CPU_DESC_FIELD(l_user_cpu_desc, i) = value;
2695 }
2696
2697 clear_set_special_reg_idx(i, UINT64_MAX, value);
2698 }
2699 }
2700
2701 /*
2702 * Updates a special register in all views. This creates a copy of the
2703 * register then clears it and sets new bits. It will then compare this
2704 * with the old version as if it was the ID register for a new CPU.
2705 *
2706 * It is intended to let code that disables features, e.g. due to errata,
2707 * to clear the user visible field.
2708 *
2709 * This needs to be called before the HWCAPs are set. If called from a CPU
2710 * feature handler this safe to call from CPU_FEAT_EARLY_BOOT. It also needs
2711 * to be before link_elf_late_ireloc is called. As this is called after the
2712 * HWCAPs are set the check for these is enough.
2713 */
2714 void
update_special_reg_iss(u_int iss,uint64_t clear,uint64_t set)2715 update_special_reg_iss(u_int iss, uint64_t clear, uint64_t set)
2716 {
2717 MPASS(hwcaps_set == false);
2718 /* There is no locking here, so we only support changing this on CPU0 */
2719 /* TODO: Add said locking */
2720 MPASS(PCPU_GET(cpuid) == 0);
2721
2722 for (int i = 0; i < nitems(user_regs); i++) {
2723 if (user_regs[i].iss != iss)
2724 continue;
2725
2726 clear_set_special_reg_idx(i, clear, set);
2727 return;
2728 }
2729 }
2730
2731 void
cpu_desc_init(void)2732 cpu_desc_init(void)
2733 {
2734 if (mp_ncpus == 1)
2735 return;
2736
2737 /*
2738 * Allocate memory for the non-boot CPUs to store their registers.
2739 * As this is indexed by CPU ID we need to allocate space for CPUs
2740 * 1 to mp_maxid. Because of this mp_maxid is already the correct
2741 * number of elements.
2742 */
2743 cpu_desc = mallocarray(mp_maxid, sizeof(*cpu_desc), M_IDENTCPU,
2744 M_ZERO | M_WAITOK);
2745 }
2746
2747 /* HWCAP */
2748 bool __read_frequently lse_supported = false;
2749
2750 bool __read_frequently icache_aliasing = false;
2751 bool __read_frequently icache_vmid = false;
2752
2753 int64_t dcache_line_size; /* The minimum D cache line size */
2754 int64_t icache_line_size; /* The minimum I cache line size */
2755 int64_t idcache_line_size; /* The minimum cache line size */
2756
2757 /*
2758 * Find the values to export to userspace as AT_HWCAP and AT_HWCAP2.
2759 */
2760 static void
parse_cpu_features(bool is64bit,struct cpu_desc * cpu_desc,u_long * hwcap,u_long * hwcap2)2761 parse_cpu_features(bool is64bit, struct cpu_desc *cpu_desc, u_long *hwcap,
2762 u_long *hwcap2)
2763 {
2764 const struct mrs_field_hwcap *hwcaps;
2765 const struct mrs_field *fields;
2766 uint64_t min, reg;
2767 u_long *cur_hwcap;
2768 int i, j, k;
2769
2770 for (i = 0; i < nitems(user_regs); i++) {
2771 if (user_regs[i].is64bit != is64bit)
2772 continue;
2773
2774 reg = CPU_DESC_FIELD(*cpu_desc, i);
2775 fields = user_regs[i].fields;
2776 for (j = 0; fields[j].type != 0; j++) {
2777 hwcaps = fields[j].hwcaps;
2778 if (hwcaps == NULL)
2779 continue;
2780
2781 for (k = 0; hwcaps[k].hwcap_id != 0; k++) {
2782 KASSERT(hwcaps[k].hwcap_id == 1 ||
2783 hwcaps[k].hwcap_id == 2,
2784 ("%s: Invalid HWCAP ID %d", __func__,
2785 hwcaps[k].hwcap_id));
2786
2787 cur_hwcap = hwcaps[k].hwcap_id == 1 ?
2788 hwcap : hwcap2;
2789 min = hwcaps[k].min;
2790
2791 /*
2792 * If the field is greater than the minimum
2793 * value we can set the hwcap;
2794 */
2795 if (mrs_field_cmp(reg, min, fields[j].shift,
2796 4, fields[j].sign) >= 0) {
2797 *cur_hwcap |= hwcaps[k].hwcap_val;
2798 }
2799 }
2800 }
2801 }
2802 }
2803
2804 static void
identify_cpu_sysinit(void * dummy __unused)2805 identify_cpu_sysinit(void *dummy __unused)
2806 {
2807 struct cpu_desc *desc, *prev_desc;
2808 int cpu;
2809 bool dic, idc;
2810
2811 dic = (allow_dic != 0);
2812 idc = (allow_idc != 0);
2813
2814 prev_desc = NULL;
2815 CPU_FOREACH(cpu) {
2816 desc = get_cpu_desc(cpu);
2817 if (cpu != 0) {
2818 check_cpu_regs(cpu, desc, prev_desc);
2819 update_special_regs(cpu);
2820 }
2821
2822 if (CTR_DIC_VAL(desc->ctr) == 0)
2823 dic = false;
2824 if (CTR_IDC_VAL(desc->ctr) == 0)
2825 idc = false;
2826 prev_desc = desc;
2827 }
2828
2829 #ifdef INVARIANTS
2830 /* Check we dont update the special registers after this point */
2831 hwcaps_set = true;
2832 #endif
2833
2834 /* Find the values to export to userspace as AT_HWCAP and AT_HWCAP2 */
2835 parse_cpu_features(true, &user_cpu_desc, &elf_hwcap, &elf_hwcap2);
2836 parse_cpu_features(true, &l_user_cpu_desc, &linux_elf_hwcap,
2837 &linux_elf_hwcap2);
2838 #ifdef COMPAT_FREEBSD32
2839 parse_cpu_features(false, &user_cpu_desc, &elf32_hwcap, &elf32_hwcap2);
2840 #endif
2841
2842 /* We export the CPUID registers */
2843 elf_hwcap |= HWCAP_CPUID;
2844 linux_elf_hwcap |= HWCAP_CPUID;
2845
2846 #ifdef COMPAT_FREEBSD32
2847 /* Set the default caps and any that need to check multiple fields */
2848 elf32_hwcap |= parse_cpu_features_hwcap32();
2849 #endif
2850
2851 if (dic && idc) {
2852 arm64_icache_sync_range = &arm64_dic_idc_icache_sync_range;
2853 if (bootverbose)
2854 printf("Enabling DIC & IDC ICache sync\n");
2855 } else if (idc) {
2856 arm64_icache_sync_range = &arm64_idc_aliasing_icache_sync_range;
2857 if (bootverbose)
2858 printf("Enabling IDC ICache sync\n");
2859 }
2860
2861 if ((elf_hwcap & HWCAP_ATOMICS) != 0) {
2862 lse_supported = true;
2863 if (bootverbose)
2864 printf("Enabling LSE atomics in the kernel\n");
2865 }
2866 #ifdef LSE_ATOMICS
2867 if (!lse_supported)
2868 panic("CPU does not support LSE atomic instructions");
2869 #endif
2870
2871 install_sys_handler(user_ctr_handler);
2872 install_sys_handler(user_idreg_handler);
2873 }
2874 /*
2875 * This needs to be after the APs have stareted as they may have errata that
2876 * means we need to mask out ID registers & that could affect hwcaps, etc.
2877 */
2878 SYSINIT(identify_cpu, SI_SUB_CONFIGURE, SI_ORDER_ANY, identify_cpu_sysinit,
2879 NULL);
2880
2881 static void
cpu_features_sysinit(void * dummy __unused)2882 cpu_features_sysinit(void *dummy __unused)
2883 {
2884 struct sbuf sb;
2885 struct cpu_desc *desc, *prev_desc;
2886 u_int cpu;
2887
2888 prev_desc = NULL;
2889 CPU_FOREACH(cpu) {
2890 desc = get_cpu_desc(cpu);
2891 print_cpu_features(cpu, desc, prev_desc);
2892 prev_desc = desc;
2893 }
2894
2895 /* Fill in cpu_model for the hw.model sysctl */
2896 sbuf_new(&sb, cpu_model, sizeof(cpu_model), SBUF_FIXEDLEN);
2897 print_cpu_midr(&sb, 0);
2898
2899 sbuf_finish(&sb);
2900 sbuf_delete(&sb);
2901
2902 free(cpu_desc, M_IDENTCPU);
2903 }
2904 /* Log features before APs are released and start printing to the dmesg. */
2905 SYSINIT(cpu_features, SI_SUB_SMP - 1, SI_ORDER_ANY, cpu_features_sysinit, NULL);
2906
2907 static void
tcr_set_e0pd1(void * arg __unused)2908 tcr_set_e0pd1(void *arg __unused)
2909 {
2910 uint64_t tcr;
2911
2912 tcr = READ_SPECIALREG(tcr_el1);
2913 tcr |= TCR_E0PD1;
2914 WRITE_SPECIALREG(tcr_el1, tcr);
2915 isb();
2916 }
2917
2918 /* Enable support for more recent architecture features */
2919 static void
cpu_feat_support(void * arg __unused)2920 cpu_feat_support(void *arg __unused)
2921 {
2922 /*
2923 * If FEAT_E0PD is supported use it to cause faults without a page
2924 * table walk if userspace tries to access kernel memory.
2925 */
2926 if (ID_AA64MMFR2_E0PD_VAL(kern_cpu_desc.id_aa64mmfr2) !=
2927 ID_AA64MMFR2_E0PD_NONE)
2928 smp_rendezvous(NULL, tcr_set_e0pd1, NULL, NULL);
2929 }
2930 SYSINIT(cpu_feat_support, SI_SUB_SMP, SI_ORDER_ANY, cpu_feat_support, NULL);
2931
2932 #ifdef COMPAT_FREEBSD32
2933 static u_long
parse_cpu_features_hwcap32(void)2934 parse_cpu_features_hwcap32(void)
2935 {
2936 u_long hwcap = HWCAP32_DEFAULT;
2937
2938 if ((MVFR1_SIMDLS_VAL(user_cpu_desc.mvfr1) >=
2939 MVFR1_SIMDLS_IMPL) &&
2940 (MVFR1_SIMDInt_VAL(user_cpu_desc.mvfr1) >=
2941 MVFR1_SIMDInt_IMPL) &&
2942 (MVFR1_SIMDSP_VAL(user_cpu_desc.mvfr1) >=
2943 MVFR1_SIMDSP_IMPL))
2944 hwcap |= HWCAP32_NEON;
2945
2946 return (hwcap);
2947 }
2948 #endif /* COMPAT_FREEBSD32 */
2949
2950 static void
print_register(struct sbuf * sb,const char * reg_name,uint64_t reg,void (* print_fields)(struct sbuf *,uint64_t,const void *),const void * arg)2951 print_register(struct sbuf *sb, const char *reg_name, uint64_t reg,
2952 void (*print_fields)(struct sbuf *, uint64_t, const void *),
2953 const void *arg)
2954 {
2955
2956 sbuf_printf(sb, "%29s = <", reg_name);
2957
2958 print_fields(sb, reg, arg);
2959
2960 sbuf_finish(sb);
2961 printf("%s>\n", sbuf_data(sb));
2962 sbuf_clear(sb);
2963 }
2964
2965 static void
print_id_fields(struct sbuf * sb,uint64_t reg,const void * arg)2966 print_id_fields(struct sbuf *sb, uint64_t reg, const void *arg)
2967 {
2968 const struct mrs_field *fields = arg;
2969 const struct mrs_field_value *fv;
2970 int field, i, j, printed;
2971
2972 #define SEP_STR ((printed++) == 0) ? "" : ","
2973 printed = 0;
2974 for (i = 0; fields[i].type != 0; i++) {
2975 fv = fields[i].values;
2976
2977 if (fv == NULL)
2978 goto next;
2979
2980 field = (reg & fields[i].mask) >> fields[i].shift;
2981 for (j = 0; fv[j].desc != NULL; j++) {
2982 if ((fv[j].value >> fields[i].shift) != field)
2983 continue;
2984
2985 if (fv[j].desc[0] != '\0')
2986 sbuf_printf(sb, "%s%s", SEP_STR, fv[j].desc);
2987 break;
2988 }
2989 if (fv[j].desc == NULL)
2990 sbuf_printf(sb, "%sUnknown %s(%x)", SEP_STR,
2991 fields[i].name, field);
2992
2993 next:
2994 reg &= ~(((1ul << fields[i].width) - 1) << fields[i].shift);
2995 }
2996
2997 if (reg != 0)
2998 sbuf_printf(sb, "%s%#lx", SEP_STR, reg);
2999 #undef SEP_STR
3000 }
3001
3002 static void
print_id_register(struct sbuf * sb,const char * reg_name,uint64_t reg,const struct mrs_field * fields)3003 print_id_register(struct sbuf *sb, const char *reg_name, uint64_t reg,
3004 const struct mrs_field *fields)
3005 {
3006
3007 print_register(sb, reg_name, reg, print_id_fields, fields);
3008 }
3009
3010 static void
print_cpu_midr(struct sbuf * sb,u_int cpu)3011 print_cpu_midr(struct sbuf *sb, u_int cpu)
3012 {
3013 const struct cpu_parts *cpu_partsp;
3014 const char *cpu_impl_name;
3015 const char *cpu_part_name;
3016 u_int midr;
3017 u_int impl_id;
3018 u_int part_id;
3019
3020 midr = pcpu_find(cpu)->pc_midr;
3021
3022 cpu_impl_name = NULL;
3023 cpu_partsp = NULL;
3024 impl_id = CPU_IMPL(midr);
3025 for (int i = 0; cpu_implementers[i].impl_name != NULL; i++) {
3026 if (impl_id == cpu_implementers[i].impl_id) {
3027 cpu_impl_name = cpu_implementers[i].impl_name;
3028 cpu_partsp = cpu_implementers[i].cpu_parts;
3029 break;
3030 }
3031 }
3032 /* Unknown implementer, so unknown part */
3033 if (cpu_impl_name == NULL) {
3034 sbuf_printf(sb, "Unknown Implementer (midr: %08x)", midr);
3035 return;
3036 }
3037
3038 KASSERT(cpu_partsp != NULL, ("%s: No parts table for implementer %s",
3039 __func__, cpu_impl_name));
3040
3041 cpu_part_name = NULL;
3042 part_id = CPU_PART(midr);
3043 for (int i = 0; cpu_partsp[i].part_name != NULL; i++) {
3044 if (part_id == cpu_partsp[i].part_id) {
3045 cpu_part_name = cpu_partsp[i].part_name;
3046 break;
3047 }
3048 }
3049 /* Known Implementer, Unknown part */
3050 if (cpu_part_name == NULL) {
3051 sbuf_printf(sb, "%s Unknown CPU r%dp%d (midr: %08x)",
3052 cpu_impl_name, CPU_VAR(midr), CPU_REV(midr), midr);
3053 return;
3054 }
3055
3056 sbuf_printf(sb, "%s %s r%dp%d", cpu_impl_name,
3057 cpu_part_name, CPU_VAR(midr), CPU_REV(midr));
3058 }
3059
3060 static void
print_cpu_cache(struct cpu_desc * desc,struct sbuf * sb,uint64_t ccs,bool icache,bool unified)3061 print_cpu_cache(struct cpu_desc *desc, struct sbuf *sb, uint64_t ccs,
3062 bool icache, bool unified)
3063 {
3064 size_t cache_size;
3065 size_t line_size;
3066
3067 /* LineSize is Log2(S) - 4. */
3068 line_size = 1 << ((ccs & CCSIDR_LineSize_MASK) + 4);
3069 /*
3070 * Calculate cache size (sets * ways * line size). There are different
3071 * formats depending on the FEAT_CCIDX bit in ID_AA64MMFR2 feature
3072 * register.
3073 */
3074 if ((desc->id_aa64mmfr2 & ID_AA64MMFR2_CCIDX_64))
3075 cache_size = (CCSIDR_NSETS_64(ccs) + 1) *
3076 (CCSIDR_ASSOC_64(ccs) + 1);
3077 else
3078 cache_size = (CCSIDR_NSETS(ccs) + 1) * (CCSIDR_ASSOC(ccs) + 1);
3079
3080 cache_size *= line_size;
3081 sbuf_printf(sb, "%zuKB (%s)", cache_size / 1024,
3082 icache ? "instruction" : unified ? "unified" : "data");
3083 }
3084
3085 static void
print_cpu_caches(struct sbuf * sb,struct cpu_desc * desc)3086 print_cpu_caches(struct sbuf *sb, struct cpu_desc *desc)
3087 {
3088 /* Print out each cache combination */
3089 uint64_t clidr;
3090 int i = 1;
3091 clidr = desc->clidr;
3092
3093 for (i = 0; (clidr & CLIDR_CTYPE_MASK) != 0; i++, clidr >>= 3) {
3094 int j = 0;
3095 int ctype_m = (clidr & CLIDR_CTYPE_MASK);
3096
3097 sbuf_printf(sb, " L%d cache: ", i + 1);
3098 if ((clidr & CLIDR_CTYPE_IO)) {
3099 print_cpu_cache(desc, sb, desc->ccsidr[i][j++], true,
3100 false);
3101 /* If there's more, add to the line. */
3102 if ((ctype_m & ~CLIDR_CTYPE_IO) != 0)
3103 sbuf_printf(sb, ", ");
3104 }
3105 if ((ctype_m & ~CLIDR_CTYPE_IO) != 0) {
3106 print_cpu_cache(desc, sb, desc->ccsidr[i][j], false,
3107 (clidr & CLIDR_CTYPE_UNIFIED));
3108 }
3109 sbuf_printf(sb, "\n");
3110
3111 }
3112 sbuf_finish(sb);
3113 printf("%s", sbuf_data(sb));
3114 }
3115
3116 static void
print_cpu_features(u_int cpu,struct cpu_desc * desc,struct cpu_desc * prev_desc)3117 print_cpu_features(u_int cpu, struct cpu_desc *desc,
3118 struct cpu_desc *prev_desc)
3119 {
3120 struct sbuf *sb;
3121
3122 sb = sbuf_new_auto();
3123 sbuf_printf(sb, "CPU%3u: ", cpu);
3124 print_cpu_midr(sb, cpu);
3125
3126 sbuf_cat(sb, " affinity:");
3127 switch(cpu_aff_levels) {
3128 default:
3129 case 4:
3130 sbuf_printf(sb, " %2d", CPU_AFF3(desc->mpidr));
3131 /* FALLTHROUGH */
3132 case 3:
3133 sbuf_printf(sb, " %2d", CPU_AFF2(desc->mpidr));
3134 /* FALLTHROUGH */
3135 case 2:
3136 sbuf_printf(sb, " %2d", CPU_AFF1(desc->mpidr));
3137 /* FALLTHROUGH */
3138 case 1:
3139 case 0: /* On UP this will be zero */
3140 sbuf_printf(sb, " %2d", CPU_AFF0(desc->mpidr));
3141 break;
3142 }
3143 sbuf_finish(sb);
3144 printf("%s\n", sbuf_data(sb));
3145 sbuf_clear(sb);
3146
3147 /*
3148 * There is a hardware errata where, if one CPU is performing a TLB
3149 * invalidation while another is performing a store-exclusive the
3150 * store-exclusive may return the wrong status. A workaround seems
3151 * to be to use an IPI to invalidate on each CPU, however given the
3152 * limited number of affected units (pass 1.1 is the evaluation
3153 * hardware revision), and the lack of information from Cavium
3154 * this has not been implemented.
3155 *
3156 * At the time of writing this the only information is from:
3157 * https://lkml.org/lkml/2016/8/4/722
3158 */
3159 /*
3160 * XXX: CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1 on its own also
3161 * triggers on pass 2.0+.
3162 */
3163 if (cpu == 0 && CPU_VAR(PCPU_GET(midr)) == 0 &&
3164 CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1)
3165 printf("WARNING: ThunderX Pass 1.1 detected.\nThis has known "
3166 "hardware bugs that may cause the incorrect operation of "
3167 "atomic operations.\n");
3168
3169 #define SHOULD_PRINT_REG(_reg) \
3170 (prev_desc == NULL || desc->_reg != prev_desc->_reg)
3171
3172 /* Cache Type Register */
3173 if (SHOULD_PRINT_REG(ctr))
3174 print_id_register(sb, "Cache Type", desc->ctr, ctr_fields);
3175
3176 /* AArch64 Instruction Set Attribute Register 0 */
3177 if (SHOULD_PRINT_REG(id_aa64isar0))
3178 print_id_register(sb, "Instruction Set Attributes 0",
3179 desc->id_aa64isar0, id_aa64isar0_fields);
3180
3181 /* AArch64 Instruction Set Attribute Register 1 */
3182 if (SHOULD_PRINT_REG(id_aa64isar1))
3183 print_id_register(sb, "Instruction Set Attributes 1",
3184 desc->id_aa64isar1, id_aa64isar1_fields);
3185
3186 /* AArch64 Instruction Set Attribute Register 2 */
3187 if (SHOULD_PRINT_REG(id_aa64isar2))
3188 print_id_register(sb, "Instruction Set Attributes 2",
3189 desc->id_aa64isar2, id_aa64isar2_fields);
3190
3191 /* AArch64 Processor Feature Register 0 */
3192 if (SHOULD_PRINT_REG(id_aa64pfr0))
3193 print_id_register(sb, "Processor Features 0",
3194 desc->id_aa64pfr0, id_aa64pfr0_fields);
3195
3196 /* AArch64 Processor Feature Register 1 */
3197 if (SHOULD_PRINT_REG(id_aa64pfr1))
3198 print_id_register(sb, "Processor Features 1",
3199 desc->id_aa64pfr1, id_aa64pfr1_fields);
3200
3201 /* AArch64 Processor Feature Register 2 */
3202 if (SHOULD_PRINT_REG(id_aa64pfr2))
3203 print_id_register(sb, "Processor Features 2",
3204 desc->id_aa64pfr2, id_aa64pfr2_fields);
3205
3206 /* AArch64 Memory Model Feature Register 0 */
3207 if (SHOULD_PRINT_REG(id_aa64mmfr0))
3208 print_id_register(sb, "Memory Model Features 0",
3209 desc->id_aa64mmfr0, id_aa64mmfr0_fields);
3210
3211 /* AArch64 Memory Model Feature Register 1 */
3212 if (SHOULD_PRINT_REG(id_aa64mmfr1))
3213 print_id_register(sb, "Memory Model Features 1",
3214 desc->id_aa64mmfr1, id_aa64mmfr1_fields);
3215
3216 /* AArch64 Memory Model Feature Register 2 */
3217 if (SHOULD_PRINT_REG(id_aa64mmfr2))
3218 print_id_register(sb, "Memory Model Features 2",
3219 desc->id_aa64mmfr2, id_aa64mmfr2_fields);
3220
3221 /* AArch64 Memory Model Feature Register 3 */
3222 if (SHOULD_PRINT_REG(id_aa64mmfr3))
3223 print_id_register(sb, "Memory Model Features 3",
3224 desc->id_aa64mmfr3, id_aa64mmfr3_fields);
3225
3226 /* AArch64 Memory Model Feature Register 4 */
3227 if (SHOULD_PRINT_REG(id_aa64mmfr4))
3228 print_id_register(sb, "Memory Model Features 4",
3229 desc->id_aa64mmfr4, id_aa64mmfr4_fields);
3230
3231 /* AArch64 Debug Feature Register 0 */
3232 if (SHOULD_PRINT_REG(id_aa64dfr0))
3233 print_id_register(sb, "Debug Features 0",
3234 desc->id_aa64dfr0, id_aa64dfr0_fields);
3235
3236 /* AArch64 Memory Model Feature Register 1 */
3237 if (SHOULD_PRINT_REG(id_aa64dfr1))
3238 print_id_register(sb, "Debug Features 1",
3239 desc->id_aa64dfr1, id_aa64dfr1_fields);
3240
3241 /* AArch64 Auxiliary Feature Register 0 */
3242 if (SHOULD_PRINT_REG(id_aa64afr0))
3243 print_id_register(sb, "Auxiliary Features 0",
3244 desc->id_aa64afr0, id_aa64afr0_fields);
3245
3246 /* AArch64 Auxiliary Feature Register 1 */
3247 if (SHOULD_PRINT_REG(id_aa64afr1))
3248 print_id_register(sb, "Auxiliary Features 1",
3249 desc->id_aa64afr1, id_aa64afr1_fields);
3250
3251 /* AArch64 SVE Feature Register 0 */
3252 if (desc->have_sve) {
3253 if (SHOULD_PRINT_REG(id_aa64zfr0) ||
3254 !prev_desc->have_sve) {
3255 print_id_register(sb, "SVE Features 0",
3256 desc->id_aa64zfr0, id_aa64zfr0_fields);
3257 }
3258 }
3259
3260 #ifdef COMPAT_FREEBSD32
3261 /* AArch32 Instruction Set Attribute Register 5 */
3262 if (SHOULD_PRINT_REG(id_isar5))
3263 print_id_register(sb, "AArch32 Instruction Set Attributes 5",
3264 desc->id_isar5, id_isar5_fields);
3265
3266 /* AArch32 Media and VFP Feature Register 0 */
3267 if (SHOULD_PRINT_REG(mvfr0))
3268 print_id_register(sb, "AArch32 Media and VFP Features 0",
3269 desc->mvfr0, mvfr0_fields);
3270
3271 /* AArch32 Media and VFP Feature Register 1 */
3272 if (SHOULD_PRINT_REG(mvfr1))
3273 print_id_register(sb, "AArch32 Media and VFP Features 1",
3274 desc->mvfr1, mvfr1_fields);
3275 #endif
3276 if (bootverbose)
3277 print_cpu_caches(sb, desc);
3278
3279 sbuf_delete(sb);
3280 sb = NULL;
3281 #undef SHOULD_PRINT_REG
3282 #undef SEP_STR
3283 }
3284
3285 void
identify_cache(uint64_t ctr)3286 identify_cache(uint64_t ctr)
3287 {
3288
3289 /* Identify the L1 cache type */
3290 switch (CTR_L1IP_VAL(ctr)) {
3291 case CTR_L1IP_PIPT:
3292 break;
3293 default:
3294 case CTR_L1IP_VIPT:
3295 icache_aliasing = true;
3296 break;
3297 }
3298
3299 if (dcache_line_size == 0) {
3300 KASSERT(icache_line_size == 0, ("%s: i-cacheline size set: %ld",
3301 __func__, icache_line_size));
3302
3303 /* Get the D cache line size */
3304 dcache_line_size = CTR_DLINE_SIZE(ctr);
3305 /* And the same for the I cache */
3306 icache_line_size = CTR_ILINE_SIZE(ctr);
3307
3308 idcache_line_size = MIN(dcache_line_size, icache_line_size);
3309 }
3310
3311 if (dcache_line_size != CTR_DLINE_SIZE(ctr)) {
3312 printf("WARNING: D-cacheline size mismatch %ld != %d\n",
3313 dcache_line_size, CTR_DLINE_SIZE(ctr));
3314 }
3315
3316 if (icache_line_size != CTR_ILINE_SIZE(ctr)) {
3317 printf("WARNING: I-cacheline size mismatch %ld != %d\n",
3318 icache_line_size, CTR_ILINE_SIZE(ctr));
3319 }
3320 }
3321
3322 void
identify_cpu(u_int cpu)3323 identify_cpu(u_int cpu)
3324 {
3325 struct cpu_desc *desc;
3326 uint64_t clidr;
3327
3328 desc = get_cpu_desc(cpu);
3329 /* Save affinity for current CPU */
3330 desc->mpidr = get_mpidr();
3331 CPU_AFFINITY(cpu) = desc->mpidr & CPU_AFF_MASK;
3332
3333 desc->ctr = READ_SPECIALREG(ctr_el0);
3334 desc->id_aa64dfr0 = READ_SPECIALREG(ID_AA64DFR0_EL1_REG);
3335 desc->id_aa64dfr1 = READ_SPECIALREG(ID_AA64DFR1_EL1_REG);
3336 desc->id_aa64isar0 = READ_SPECIALREG(ID_AA64ISAR0_EL1_REG);
3337 desc->id_aa64isar1 = READ_SPECIALREG(ID_AA64ISAR1_EL1_REG);
3338 desc->id_aa64isar2 = READ_SPECIALREG(ID_AA64ISAR2_EL1_REG);
3339 desc->id_aa64mmfr0 = READ_SPECIALREG(ID_AA64MMFR0_EL1_REG);
3340 desc->id_aa64mmfr1 = READ_SPECIALREG(ID_AA64MMFR1_EL1_REG);
3341 desc->id_aa64mmfr2 = READ_SPECIALREG(ID_AA64MMFR2_EL1_REG);
3342 desc->id_aa64mmfr3 = READ_SPECIALREG(ID_AA64MMFR3_EL1_REG);
3343 desc->id_aa64mmfr4 = READ_SPECIALREG(ID_AA64MMFR4_EL1_REG);
3344 desc->id_aa64pfr0 = READ_SPECIALREG(ID_AA64PFR0_EL1_REG);
3345 desc->id_aa64pfr1 = READ_SPECIALREG(ID_AA64PFR1_EL1_REG);
3346 desc->id_aa64pfr2 = READ_SPECIALREG(ID_AA64PFR2_EL1_REG);
3347
3348 /*
3349 * ID_AA64ZFR0_EL1 is only valid when at least one of:
3350 * - ID_AA64PFR0_EL1.SVE is non-zero
3351 * - ID_AA64PFR1_EL1.SME is non-zero
3352 * In other cases it is zero, but still safe to read
3353 */
3354 desc->have_sve =
3355 (ID_AA64PFR0_SVE_VAL(desc->id_aa64pfr0) != 0);
3356 desc->id_aa64zfr0 = READ_SPECIALREG(ID_AA64ZFR0_EL1_REG);
3357
3358 desc->clidr = READ_SPECIALREG(clidr_el1);
3359
3360 clidr = desc->clidr;
3361
3362 for (int i = 0; (clidr & CLIDR_CTYPE_MASK) != 0; i++, clidr >>= 3) {
3363 int j = 0;
3364 if ((clidr & CLIDR_CTYPE_IO)) {
3365 WRITE_SPECIALREG(csselr_el1,
3366 CSSELR_Level(i) | CSSELR_InD);
3367 desc->ccsidr[i][j++] =
3368 READ_SPECIALREG(ccsidr_el1);
3369 }
3370 if ((clidr & ~CLIDR_CTYPE_IO) == 0)
3371 continue;
3372 WRITE_SPECIALREG(csselr_el1, CSSELR_Level(i));
3373 desc->ccsidr[i][j] = READ_SPECIALREG(ccsidr_el1);
3374 }
3375
3376 #ifdef COMPAT_FREEBSD32
3377 /* Only read aarch32 SRs if EL0-32 is available */
3378 if (ID_AA64PFR0_EL0_VAL(desc->id_aa64pfr0) == ID_AA64PFR0_EL0_64_32) {
3379 desc->id_isar5 = READ_SPECIALREG(id_isar5_el1);
3380 desc->mvfr0 = READ_SPECIALREG(mvfr0_el1);
3381 desc->mvfr1 = READ_SPECIALREG(mvfr1_el1);
3382 }
3383 #endif
3384 }
3385
3386 static void
check_cpu_regs(u_int cpu,struct cpu_desc * desc,struct cpu_desc * prev_desc)3387 check_cpu_regs(u_int cpu, struct cpu_desc *desc, struct cpu_desc *prev_desc)
3388 {
3389 switch (cpu_aff_levels) {
3390 case 0:
3391 if (CPU_AFF0(desc->mpidr) != CPU_AFF0(prev_desc->mpidr))
3392 cpu_aff_levels = 1;
3393 /* FALLTHROUGH */
3394 case 1:
3395 if (CPU_AFF1(desc->mpidr) != CPU_AFF1(prev_desc->mpidr))
3396 cpu_aff_levels = 2;
3397 /* FALLTHROUGH */
3398 case 2:
3399 if (CPU_AFF2(desc->mpidr) != CPU_AFF2(prev_desc->mpidr))
3400 cpu_aff_levels = 3;
3401 /* FALLTHROUGH */
3402 case 3:
3403 if (CPU_AFF3(desc->mpidr) != CPU_AFF3(prev_desc->mpidr))
3404 cpu_aff_levels = 4;
3405 break;
3406 }
3407
3408 if (desc->ctr != prev_desc->ctr) {
3409 /*
3410 * If the cache is different on different cores we should
3411 * emulate for userspace to provide a uniform value
3412 */
3413 emulate_ctr = true;
3414
3415 /*
3416 * If the cache type register is different we may
3417 * have a different l1 cache type.
3418 */
3419 identify_cache(desc->ctr);
3420 }
3421 }
3422