1 /*- 2 * Copyright (c) 2014 Andrew Turner 3 * Copyright (c) 2014 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * Portions of this software were developed by Semihalf 7 * under sponsorship of the FreeBSD Foundation. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 */ 31 32 #include <sys/param.h> 33 #include <sys/kernel.h> 34 #include <sys/malloc.h> 35 #include <sys/proc.h> 36 #include <sys/pcpu.h> 37 #include <sys/sbuf.h> 38 #include <sys/smp.h> 39 #include <sys/sysctl.h> 40 #include <sys/sysent.h> 41 #include <sys/systm.h> 42 43 #include <machine/atomic.h> 44 #include <machine/cpu.h> 45 #include <machine/cpu_feat.h> 46 #include <machine/cpufunc.h> 47 #include <machine/elf.h> 48 #include <machine/md_var.h> 49 #include <machine/undefined.h> 50 51 static MALLOC_DEFINE(M_IDENTCPU, "CPU ID", "arm64 CPU identification memory"); 52 53 struct cpu_desc; 54 #ifdef INVARIANTS 55 static bool hwcaps_set = false; 56 #endif 57 58 static void print_cpu_midr(struct sbuf *sb, u_int cpu); 59 static void print_cpu_features(u_int cpu, struct cpu_desc *desc, 60 struct cpu_desc *prev_desc); 61 static void print_cpu_caches(struct sbuf *sb, struct cpu_desc *desc); 62 #ifdef COMPAT_FREEBSD32 63 static u_long parse_cpu_features_hwcap32(void); 64 #endif 65 66 const char machine[] = "arm64"; 67 68 #ifdef SCTL_MASK32 69 extern int adaptive_machine_arch; 70 #endif 71 72 static SYSCTL_NODE(_machdep, OID_AUTO, cache, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 73 "Cache management tuning"); 74 75 static int allow_dic = 1; 76 SYSCTL_INT(_machdep_cache, OID_AUTO, allow_dic, CTLFLAG_RDTUN, &allow_dic, 0, 77 "Allow optimizations based on the DIC cache bit"); 78 79 static int allow_idc = 1; 80 SYSCTL_INT(_machdep_cache, OID_AUTO, allow_idc, CTLFLAG_RDTUN, &allow_idc, 0, 81 "Allow optimizations based on the IDC cache bit"); 82 83 static bool emulate_ctr = false; 84 85 static void check_cpu_regs(u_int cpu, struct cpu_desc *desc, 86 struct cpu_desc *prev_desc); 87 static uint64_t update_special_reg_field(uint64_t user_reg, u_int type, 88 uint64_t value, u_int width, u_int shift, bool sign); 89 90 /* 91 * The default implementation of I-cache sync assumes we have an 92 * aliasing cache until we know otherwise. 93 */ 94 void (*arm64_icache_sync_range)(void *, vm_size_t) = 95 &arm64_aliasing_icache_sync_range; 96 97 static int 98 sysctl_hw_machine(SYSCTL_HANDLER_ARGS) 99 { 100 #ifdef SCTL_MASK32 101 static const char machine32[] = "arm"; 102 #endif 103 int error; 104 105 #ifdef SCTL_MASK32 106 if ((req->flags & SCTL_MASK32) != 0 && adaptive_machine_arch) 107 error = SYSCTL_OUT(req, machine32, sizeof(machine32)); 108 else 109 #endif 110 error = SYSCTL_OUT(req, machine, sizeof(machine)); 111 return (error); 112 } 113 114 SYSCTL_PROC(_hw, HW_MACHINE, machine, CTLTYPE_STRING | CTLFLAG_RD | 115 CTLFLAG_CAPRD | CTLFLAG_MPSAFE, NULL, 0, sysctl_hw_machine, "A", 116 "Machine class"); 117 118 static char cpu_model[64]; 119 SYSCTL_STRING(_hw, HW_MODEL, model, CTLFLAG_RD | CTLFLAG_CAPRD, 120 cpu_model, sizeof(cpu_model), "Machine model"); 121 122 #define MAX_CACHES 8 /* Maximum number of caches supported 123 architecturally. */ 124 /* 125 * Per-CPU affinity as provided in MPIDR_EL1 126 * Indexed by CPU number in logical order selected by the system. 127 * Relevant fields can be extracted using CPU_AFFn macros, 128 * Aff3.Aff2.Aff1.Aff0 construct a unique CPU address in the system. 129 * 130 * Fields used by us: 131 * Aff1 - Cluster number 132 * Aff0 - CPU number in Aff1 cluster 133 */ 134 uint64_t __cpu_affinity[MAXCPU]; 135 static u_int cpu_aff_levels; 136 137 struct cpu_desc { 138 uint64_t mpidr; 139 uint64_t id_aa64afr0; 140 uint64_t id_aa64afr1; 141 uint64_t id_aa64dfr0; 142 uint64_t id_aa64dfr1; 143 uint64_t id_aa64isar0; 144 uint64_t id_aa64isar1; 145 uint64_t id_aa64isar2; 146 uint64_t id_aa64mmfr0; 147 uint64_t id_aa64mmfr1; 148 uint64_t id_aa64mmfr2; 149 uint64_t id_aa64mmfr3; 150 uint64_t id_aa64mmfr4; 151 uint64_t id_aa64pfr0; 152 uint64_t id_aa64pfr1; 153 uint64_t id_aa64pfr2; 154 uint64_t id_aa64zfr0; 155 uint64_t ctr; 156 #ifdef COMPAT_FREEBSD32 157 uint64_t id_isar5; 158 uint64_t mvfr0; 159 uint64_t mvfr1; 160 #endif 161 uint64_t clidr; 162 uint32_t ccsidr[MAX_CACHES][2]; /* 2 possible types. */ 163 bool have_sve; 164 }; 165 166 static struct cpu_desc cpu_desc0; 167 static struct cpu_desc *cpu_desc; 168 static struct cpu_desc kern_cpu_desc; 169 static struct cpu_desc user_cpu_desc; 170 static struct cpu_desc l_user_cpu_desc; 171 172 static struct cpu_desc * 173 get_cpu_desc(u_int cpu) 174 { 175 /* The cpu_desc for CPU 0 is used before the allocator is ready. */ 176 if (cpu == 0) 177 return (&cpu_desc0); 178 179 MPASS(cpu_desc != NULL); 180 return (&cpu_desc[cpu - 1]); 181 } 182 183 struct cpu_parts { 184 u_int part_id; 185 const char *part_name; 186 }; 187 #define CPU_PART_NONE { 0, NULL } 188 189 struct cpu_implementers { 190 u_int impl_id; 191 const char *impl_name; 192 /* 193 * Part number is implementation defined 194 * so each vendor will have its own set of values and names. 195 */ 196 const struct cpu_parts *cpu_parts; 197 }; 198 #define CPU_IMPLEMENTER_NONE { 0, NULL, NULL } 199 200 /* 201 * Per-implementer table of (PartNum, CPU Name) pairs. 202 */ 203 /* ARM Ltd. */ 204 static const struct cpu_parts cpu_parts_arm[] = { 205 { CPU_PART_AEM_V8, "AEMv8" }, 206 { CPU_PART_FOUNDATION, "Foundation-Model" }, 207 { CPU_PART_CORTEX_A34, "Cortex-A34" }, 208 { CPU_PART_CORTEX_A35, "Cortex-A35" }, 209 { CPU_PART_CORTEX_A53, "Cortex-A53" }, 210 { CPU_PART_CORTEX_A55, "Cortex-A55" }, 211 { CPU_PART_CORTEX_A57, "Cortex-A57" }, 212 { CPU_PART_CORTEX_A65, "Cortex-A65" }, 213 { CPU_PART_CORTEX_A65AE, "Cortex-A65AE" }, 214 { CPU_PART_CORTEX_A72, "Cortex-A72" }, 215 { CPU_PART_CORTEX_A73, "Cortex-A73" }, 216 { CPU_PART_CORTEX_A75, "Cortex-A75" }, 217 { CPU_PART_CORTEX_A76, "Cortex-A76" }, 218 { CPU_PART_CORTEX_A76AE, "Cortex-A76AE" }, 219 { CPU_PART_CORTEX_A77, "Cortex-A77" }, 220 { CPU_PART_CORTEX_A78, "Cortex-A78" }, 221 { CPU_PART_CORTEX_A78AE, "Cortex-A78AE" }, 222 { CPU_PART_CORTEX_A78C, "Cortex-A78C" }, 223 { CPU_PART_CORTEX_A510, "Cortex-A510" }, 224 { CPU_PART_CORTEX_A520, "Cortex-A520" }, 225 { CPU_PART_CORTEX_A710, "Cortex-A710" }, 226 { CPU_PART_CORTEX_A715, "Cortex-A715" }, 227 { CPU_PART_CORTEX_A720, "Cortex-A720" }, 228 { CPU_PART_CORTEX_A725, "Cortex-A725" }, 229 { CPU_PART_CORTEX_X925, "Cortex-A925" }, 230 { CPU_PART_CORTEX_X1, "Cortex-X1" }, 231 { CPU_PART_CORTEX_X1C, "Cortex-X1C" }, 232 { CPU_PART_CORTEX_X2, "Cortex-X2" }, 233 { CPU_PART_CORTEX_X3, "Cortex-X3" }, 234 { CPU_PART_CORTEX_X4, "Cortex-X4" }, 235 { CPU_PART_C1_NANO, "C1-Nano" }, 236 { CPU_PART_C1_PRO, "C1-Pro" }, 237 { CPU_PART_C1_PREMIUM, "C1-Premium" }, 238 { CPU_PART_C1_ULTRA, "C1-Ultra" }, 239 { CPU_PART_NEOVERSE_E1, "Neoverse-E1" }, 240 { CPU_PART_NEOVERSE_N1, "Neoverse-N1" }, 241 { CPU_PART_NEOVERSE_N2, "Neoverse-N2" }, 242 { CPU_PART_NEOVERSE_N3, "Neoverse-N3" }, 243 { CPU_PART_NEOVERSE_V1, "Neoverse-V1" }, 244 { CPU_PART_NEOVERSE_V2, "Neoverse-V2" }, 245 { CPU_PART_NEOVERSE_V3, "Neoverse-V3" }, 246 { CPU_PART_NEOVERSE_V3AE, "Neoverse-V3AE" }, 247 CPU_PART_NONE, 248 }; 249 250 /* Cavium */ 251 static const struct cpu_parts cpu_parts_cavium[] = { 252 { CPU_PART_THUNDERX, "ThunderX" }, 253 { CPU_PART_THUNDERX2, "ThunderX2" }, 254 CPU_PART_NONE, 255 }; 256 257 /* APM (now Ampere) */ 258 static const struct cpu_parts cpu_parts_apm[] = { 259 { CPU_PART_EMAG8180, "eMAG 8180" }, 260 CPU_PART_NONE, 261 }; 262 263 /* Ampere */ 264 static const struct cpu_parts cpu_parts_ampere[] = { 265 { CPU_PART_AMPERE1, "AmpereOne AC03" }, 266 { CPU_PART_AMPERE1A, "AmpereOne AC04" }, 267 CPU_PART_NONE, 268 }; 269 270 /* Microsoft */ 271 static const struct cpu_parts cpu_parts_microsoft[] = { 272 { CPU_PART_AZURE_COBALT_100, "Azure Cobalt 100" }, 273 CPU_PART_NONE, 274 }; 275 276 /* Qualcomm */ 277 static const struct cpu_parts cpu_parts_qcom[] = { 278 { CPU_PART_KRYO400_GOLD, "Kryo 400 Gold" }, 279 { CPU_PART_KRYO400_SILVER, "Kryo 400 Silver" }, 280 CPU_PART_NONE, 281 }; 282 283 /* Apple */ 284 static const struct cpu_parts cpu_parts_apple[] = { 285 { CPU_PART_M1_ICESTORM, "M1 Icestorm" }, 286 { CPU_PART_M1_FIRESTORM, "M1 Firestorm" }, 287 { CPU_PART_M1_ICESTORM_PRO, "M1 Pro Icestorm" }, 288 { CPU_PART_M1_FIRESTORM_PRO, "M1 Pro Firestorm" }, 289 { CPU_PART_M1_ICESTORM_MAX, "M1 Max Icestorm" }, 290 { CPU_PART_M1_FIRESTORM_MAX, "M1 Max Firestorm" }, 291 { CPU_PART_M2_BLIZZARD, "M2 Blizzard" }, 292 { CPU_PART_M2_AVALANCHE, "M2 Avalanche" }, 293 { CPU_PART_M2_BLIZZARD_PRO, "M2 Pro Blizzard" }, 294 { CPU_PART_M2_AVALANCHE_PRO, "M2 Pro Avalanche" }, 295 { CPU_PART_M2_BLIZZARD_MAX, "M2 Max Blizzard" }, 296 { CPU_PART_M2_AVALANCHE_MAX, "M2 Max Avalanche" }, 297 CPU_PART_NONE, 298 }; 299 300 /* Unknown */ 301 static const struct cpu_parts cpu_parts_none[] = { 302 CPU_PART_NONE, 303 }; 304 305 /* 306 * Implementers table. 307 */ 308 const struct cpu_implementers cpu_implementers[] = { 309 { CPU_IMPL_AMPERE, "Ampere", cpu_parts_ampere }, 310 { CPU_IMPL_APPLE, "Apple", cpu_parts_apple }, 311 { CPU_IMPL_APM, "APM", cpu_parts_apm }, 312 { CPU_IMPL_ARM, "ARM", cpu_parts_arm }, 313 { CPU_IMPL_BROADCOM, "Broadcom", cpu_parts_none }, 314 { CPU_IMPL_CAVIUM, "Cavium", cpu_parts_cavium }, 315 { CPU_IMPL_DEC, "DEC", cpu_parts_none }, 316 { CPU_IMPL_FREESCALE, "Freescale", cpu_parts_none }, 317 { CPU_IMPL_FUJITSU, "Fujitsu", cpu_parts_none }, 318 { CPU_IMPL_HISILICON, "HiSilicon", cpu_parts_none }, 319 { CPU_IMPL_INFINEON, "IFX", cpu_parts_none }, 320 { CPU_IMPL_INTEL, "Intel", cpu_parts_none }, 321 { CPU_IMPL_MARVELL, "Marvell", cpu_parts_none }, 322 { CPU_IMPL_MICROSOFT, "Microsoft", cpu_parts_microsoft }, 323 { CPU_IMPL_NVIDIA, "NVIDIA", cpu_parts_none }, 324 { CPU_IMPL_QUALCOMM, "Qualcomm", cpu_parts_qcom }, 325 CPU_IMPLEMENTER_NONE, 326 }; 327 328 #define MRS_TYPE_MASK 0xf 329 #define MRS_INVALID 0 330 #define MRS_EXACT 1 331 #define MRS_EXACT_IF_DIFFERENT 2 332 #define MRS_LOWER 3 333 #define MRS_HIGHER_OR_ZERO 4 334 #define MRS_HIGHER 5 335 #define MRS_SAFE_SHIFT 4 336 #define MRS_SAFE_MASK (0xfu << MRS_SAFE_SHIFT) 337 #define MRS_SAFE(x) (((x) << MRS_SAFE_SHIFT) & MRS_SAFE_MASK) 338 #define MRS_SAFE_VAL(x) (((x) & MRS_SAFE_MASK) >> MRS_SAFE_SHIFT) 339 #define MRS_FREEBSD (1u << 8) 340 #define MRS_LINUX (1u << 9) 341 #define MRS_USERSPACE (MRS_FREEBSD | MRS_LINUX) 342 343 struct mrs_field_value { 344 uint64_t value; 345 const char *desc; 346 }; 347 348 #define MRS_FIELD_VALUE(_value, _desc) \ 349 { \ 350 .value = (_value), \ 351 .desc = (_desc), \ 352 } 353 354 #define MRS_FIELD_VALUE_NONE_IMPL(_reg, _field, _none, _impl) \ 355 MRS_FIELD_VALUE(_reg ## _ ## _field ## _ ## _none, ""), \ 356 MRS_FIELD_VALUE(_reg ## _ ## _field ## _ ## _impl, #_field) 357 358 #define MRS_FIELD_VALUE_COUNT(_reg, _field, _desc) \ 359 MRS_FIELD_VALUE(0ul << _reg ## _ ## _field ## _SHIFT, "1 " _desc), \ 360 MRS_FIELD_VALUE(1ul << _reg ## _ ## _field ## _SHIFT, "2 " _desc "s"), \ 361 MRS_FIELD_VALUE(2ul << _reg ## _ ## _field ## _SHIFT, "3 " _desc "s"), \ 362 MRS_FIELD_VALUE(3ul << _reg ## _ ## _field ## _SHIFT, "4 " _desc "s"), \ 363 MRS_FIELD_VALUE(4ul << _reg ## _ ## _field ## _SHIFT, "5 " _desc "s"), \ 364 MRS_FIELD_VALUE(5ul << _reg ## _ ## _field ## _SHIFT, "6 " _desc "s"), \ 365 MRS_FIELD_VALUE(6ul << _reg ## _ ## _field ## _SHIFT, "7 " _desc "s"), \ 366 MRS_FIELD_VALUE(7ul << _reg ## _ ## _field ## _SHIFT, "8 " _desc "s"), \ 367 MRS_FIELD_VALUE(8ul << _reg ## _ ## _field ## _SHIFT, "9 " _desc "s"), \ 368 MRS_FIELD_VALUE(9ul << _reg ## _ ## _field ## _SHIFT, "10 "_desc "s"), \ 369 MRS_FIELD_VALUE(10ul<< _reg ## _ ## _field ## _SHIFT, "11 "_desc "s"), \ 370 MRS_FIELD_VALUE(11ul<< _reg ## _ ## _field ## _SHIFT, "12 "_desc "s"), \ 371 MRS_FIELD_VALUE(12ul<< _reg ## _ ## _field ## _SHIFT, "13 "_desc "s"), \ 372 MRS_FIELD_VALUE(13ul<< _reg ## _ ## _field ## _SHIFT, "14 "_desc "s"), \ 373 MRS_FIELD_VALUE(14ul<< _reg ## _ ## _field ## _SHIFT, "15 "_desc "s"), \ 374 MRS_FIELD_VALUE(15ul<< _reg ## _ ## _field ## _SHIFT, "16 "_desc "s") 375 376 /* 377 * Used for printing I/D cache line sizes & CWG/ERG, as 0 is a special case 378 * in some cases the decoded string needs to be passed in. 379 */ 380 #define MRS_FIELD_VALUE_CACHE(_reg, _field, _0desc, _desc) \ 381 MRS_FIELD_VALUE(0ul << _reg ## _ ## _field ## _SHIFT, _0desc), \ 382 MRS_FIELD_VALUE(1ul << _reg ## _ ## _field ## _SHIFT, "8 " _desc), \ 383 MRS_FIELD_VALUE(2ul << _reg ## _ ## _field ## _SHIFT, "16 " _desc), \ 384 MRS_FIELD_VALUE(3ul << _reg ## _ ## _field ## _SHIFT, "32 " _desc), \ 385 MRS_FIELD_VALUE(4ul << _reg ## _ ## _field ## _SHIFT, "64 " _desc), \ 386 MRS_FIELD_VALUE(5ul << _reg ## _ ## _field ## _SHIFT, "128 " _desc), \ 387 MRS_FIELD_VALUE(6ul << _reg ## _ ## _field ## _SHIFT, "256 " _desc), \ 388 MRS_FIELD_VALUE(7ul << _reg ## _ ## _field ## _SHIFT, "512 " _desc), \ 389 MRS_FIELD_VALUE(8ul << _reg ## _ ## _field ## _SHIFT, "1k " _desc), \ 390 MRS_FIELD_VALUE(9ul << _reg ## _ ## _field ## _SHIFT, "2k " _desc), \ 391 MRS_FIELD_VALUE(10ul<< _reg ## _ ## _field ## _SHIFT, "4k " _desc), \ 392 MRS_FIELD_VALUE(11ul<< _reg ## _ ## _field ## _SHIFT, "8k " _desc), \ 393 MRS_FIELD_VALUE(12ul<< _reg ## _ ## _field ## _SHIFT, "16k " _desc), \ 394 MRS_FIELD_VALUE(13ul<< _reg ## _ ## _field ## _SHIFT, "32k " _desc), \ 395 MRS_FIELD_VALUE(14ul<< _reg ## _ ## _field ## _SHIFT, "64k " _desc), \ 396 MRS_FIELD_VALUE(15ul<< _reg ## _ ## _field ## _SHIFT, "128k "_desc) 397 398 #define MRS_FIELD_VALUE_END { .desc = NULL } 399 400 struct mrs_field_hwcap { 401 uint64_t min; 402 u_long hwcap_val; 403 u_int hwcap_id; 404 }; 405 406 #define MRS_HWCAP(_hwcap_id, _val, _min) \ 407 { \ 408 .hwcap_id = (_hwcap_id), \ 409 .hwcap_val = (_val), \ 410 .min = (_min), \ 411 } 412 413 #define MRS_HWCAP_END { .hwcap_id = 0 } 414 415 struct mrs_field { 416 const char *name; 417 const struct mrs_field_value *values; 418 const struct mrs_field_hwcap *hwcaps; 419 uint64_t mask; 420 bool sign; 421 u_int type; 422 u_int width; 423 u_int shift; 424 }; 425 426 #define MRS_FIELD_RES1(_width, _shift) \ 427 { \ 428 .sign = false, \ 429 .type = MRS_EXACT | MRS_SAFE((1u << (_width)) - 1) | \ 430 MRS_USERSPACE, \ 431 .width = (_width), \ 432 .shift = (_shift), \ 433 } 434 435 #define MRS_FIELD_HWCAP(_register, _name, _sign, _type, _visibility, \ 436 _values, _hwcap) \ 437 { \ 438 .name = #_name, \ 439 .sign = (_sign), \ 440 .type = ((_type) | (_visibility)), \ 441 .width = _register ## _ ## _name ## _WIDTH, \ 442 .shift = _register ## _ ## _name ## _SHIFT, \ 443 .mask = _register ## _ ## _name ## _MASK, \ 444 .values = (_values), \ 445 .hwcaps = (_hwcap), \ 446 } 447 448 #define MRS_FIELD(_register, _name, _sign, _type, _visibility, _values) \ 449 MRS_FIELD_HWCAP(_register, _name, _sign, _type, _visibility, \ 450 _values, NULL) 451 452 #define MRS_FIELD_END { .type = MRS_INVALID, } 453 454 /* CTR_EL0 */ 455 static const struct mrs_field_value ctr_dic[] = { 456 MRS_FIELD_VALUE_NONE_IMPL(CTR, DIC, NONE, IMPL), 457 MRS_FIELD_VALUE_END, 458 }; 459 460 static const struct mrs_field_value ctr_idc[] = { 461 MRS_FIELD_VALUE_NONE_IMPL(CTR, IDC, NONE, IMPL), 462 MRS_FIELD_VALUE_END, 463 }; 464 465 static const struct mrs_field_value ctr_cwg[] = { 466 MRS_FIELD_VALUE_CACHE(CTR, CWG, "Unknown CWG", 467 "byte CWG"), 468 MRS_FIELD_VALUE_END, 469 }; 470 471 static const struct mrs_field_value ctr_erg[] = { 472 MRS_FIELD_VALUE_CACHE(CTR, ERG, "Unknown ERG", 473 "byte ERG"), 474 MRS_FIELD_VALUE_END, 475 }; 476 477 static const struct mrs_field_value ctr_dline[] = { 478 MRS_FIELD_VALUE_CACHE(CTR, DLINE, "4 byte D-cacheline", 479 "byte D-cacheline"), 480 MRS_FIELD_VALUE_END, 481 }; 482 483 static const struct mrs_field_value ctr_l1ip[] = { 484 MRS_FIELD_VALUE(CTR_L1IP_VIPT, "VIPT I-cache"), 485 MRS_FIELD_VALUE(CTR_L1IP_PIPT, "PIPT I-cache"), 486 MRS_FIELD_VALUE_END, 487 }; 488 489 static const struct mrs_field_value ctr_iline[] = { 490 MRS_FIELD_VALUE_CACHE(CTR, ILINE, "4 byte I-cacheline", 491 "byte I-cacheline"), 492 MRS_FIELD_VALUE_END, 493 }; 494 495 static const struct mrs_field ctr_fields[] = { 496 /* Bit 31 is RES1 */ 497 MRS_FIELD_RES1(1, 31), 498 MRS_FIELD(CTR, DIC, false, MRS_LOWER, MRS_USERSPACE, ctr_dic), 499 MRS_FIELD(CTR, IDC, false, MRS_LOWER, MRS_USERSPACE, ctr_idc), 500 MRS_FIELD(CTR, CWG, false, MRS_HIGHER_OR_ZERO, MRS_USERSPACE, ctr_cwg), 501 MRS_FIELD(CTR, ERG, false, MRS_HIGHER_OR_ZERO, MRS_USERSPACE, ctr_erg), 502 MRS_FIELD(CTR, DLINE, false, MRS_LOWER, MRS_USERSPACE, ctr_dline), 503 /* If the ICache types are different report the safe option */ 504 MRS_FIELD(CTR, L1IP, false, MRS_EXACT_IF_DIFFERENT | 505 MRS_SAFE(CTR_L1IP_VIPT >> CTR_L1IP_SHIFT), MRS_USERSPACE, 506 ctr_l1ip), 507 MRS_FIELD(CTR, ILINE, false, MRS_LOWER, MRS_USERSPACE, ctr_iline), 508 MRS_FIELD_END, 509 }; 510 511 /* ID_AA64AFR0_EL1 */ 512 static const struct mrs_field id_aa64afr0_fields[] = { 513 MRS_FIELD_END, 514 }; 515 516 517 /* ID_AA64AFR1_EL1 */ 518 static const struct mrs_field id_aa64afr1_fields[] = { 519 MRS_FIELD_END, 520 }; 521 522 523 /* ID_AA64DFR0_EL1 */ 524 static const struct mrs_field_value id_aa64dfr0_hpmn0[] = { 525 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR0, HPMN0, CONSTR, DEFINED), 526 MRS_FIELD_VALUE_END, 527 }; 528 529 static const struct mrs_field_value id_aa64dfr0_brbe[] = { 530 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR0, BRBE, NONE, IMPL), 531 MRS_FIELD_VALUE(ID_AA64DFR0_BRBE_EL3, "BRBE EL3"), 532 MRS_FIELD_VALUE_END, 533 }; 534 535 static const struct mrs_field_value id_aa64dfr0_mtpmu[] = { 536 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR0, MTPMU, NONE, IMPL), 537 MRS_FIELD_VALUE(ID_AA64DFR0_MTPMU_NONE_MT_RES0, "MTPMU res0"), 538 MRS_FIELD_VALUE_END, 539 }; 540 541 static const struct mrs_field_value id_aa64dfr0_tracebuffer[] = { 542 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR0, TraceBuffer, NONE, IMPL), 543 MRS_FIELD_VALUE_END, 544 }; 545 546 static const struct mrs_field_value id_aa64dfr0_tracefilt[] = { 547 MRS_FIELD_VALUE(ID_AA64DFR0_TraceFilt_NONE, ""), 548 MRS_FIELD_VALUE(ID_AA64DFR0_TraceFilt_8_4, "Trace v8.4"), 549 MRS_FIELD_VALUE_END, 550 }; 551 552 static const struct mrs_field_value id_aa64dfr0_doublelock[] = { 553 MRS_FIELD_VALUE(ID_AA64DFR0_DoubleLock_IMPL, "DoubleLock"), 554 MRS_FIELD_VALUE(ID_AA64DFR0_DoubleLock_NONE, ""), 555 MRS_FIELD_VALUE_END, 556 }; 557 558 static const struct mrs_field_value id_aa64dfr0_pmsver[] = { 559 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_NONE, ""), 560 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE, "SPE"), 561 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE_1_1, "SPEv1p1"), 562 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE_1_2, "SPEv1p2"), 563 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE_1_3, "SPEv1p3"), 564 MRS_FIELD_VALUE(ID_AA64DFR0_PMSVer_SPE_1_4, "SPEv1p4"), 565 MRS_FIELD_VALUE_END, 566 }; 567 568 static const struct mrs_field_value id_aa64dfr0_ctx_cmps[] = { 569 MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, CTX_CMPs, "CTX BKPT"), 570 MRS_FIELD_VALUE_END, 571 }; 572 573 static const struct mrs_field_value id_aa64dfr0_wrps[] = { 574 MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, WRPs, "Watchpoint"), 575 MRS_FIELD_VALUE_END, 576 }; 577 578 static const struct mrs_field_value id_aa64dfr0_pmss[] = { 579 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR0, PMSS, NONE, IMPL), 580 MRS_FIELD_VALUE_END, 581 }; 582 583 static const struct mrs_field_value id_aa64dfr0_brps[] = { 584 MRS_FIELD_VALUE_COUNT(ID_AA64DFR0, BRPs, "Breakpoint"), 585 MRS_FIELD_VALUE_END, 586 }; 587 588 static const struct mrs_field_value id_aa64dfr0_pmuver[] = { 589 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_NONE, ""), 590 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3, "PMUv3"), 591 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_1, "PMUv3p1"), 592 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_4, "PMUv3p4"), 593 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_5, "PMUv3p5"), 594 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_7, "PMUv3p7"), 595 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_8, "PMUv3p8"), 596 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_3_9, "PMUv3p9"), 597 MRS_FIELD_VALUE(ID_AA64DFR0_PMUVer_IMPL, "IMPL PMU"), 598 MRS_FIELD_VALUE_END, 599 }; 600 601 static const struct mrs_field_value id_aa64dfr0_tracever[] = { 602 MRS_FIELD_VALUE(ID_AA64DFR0_TraceVer_NONE, ""), 603 MRS_FIELD_VALUE(ID_AA64DFR0_TraceVer_IMPL, "Trace"), 604 MRS_FIELD_VALUE_END, 605 }; 606 607 static const struct mrs_field_value id_aa64dfr0_debugver[] = { 608 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8, "Debugv8"), 609 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_VHE, "Debugv8_VHE"), 610 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_2, "Debugv8p2"), 611 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_4, "Debugv8p4"), 612 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_8, "Debugv8p8"), 613 MRS_FIELD_VALUE(ID_AA64DFR0_DebugVer_8_9, "Debugv8p9"), 614 MRS_FIELD_VALUE_END, 615 }; 616 617 static const struct mrs_field id_aa64dfr0_fields[] = { 618 MRS_FIELD(ID_AA64DFR0, HPMN0, false, MRS_LOWER, 0, id_aa64dfr0_hpmn0), 619 MRS_FIELD(ID_AA64DFR0, BRBE, false, MRS_LOWER, 0, id_aa64dfr0_brbe), 620 MRS_FIELD(ID_AA64DFR0, MTPMU, true, MRS_LOWER, 0, id_aa64dfr0_mtpmu), 621 MRS_FIELD(ID_AA64DFR0, TraceBuffer, false, MRS_LOWER, 0, 622 id_aa64dfr0_tracebuffer), 623 MRS_FIELD(ID_AA64DFR0, TraceFilt, false, MRS_LOWER, 0, 624 id_aa64dfr0_tracefilt), 625 MRS_FIELD(ID_AA64DFR0, DoubleLock, true, MRS_LOWER, 0, 626 id_aa64dfr0_doublelock), 627 MRS_FIELD(ID_AA64DFR0, PMSVer, false, MRS_LOWER, 0, id_aa64dfr0_pmsver), 628 MRS_FIELD(ID_AA64DFR0, CTX_CMPs, false, MRS_LOWER, 0, 629 id_aa64dfr0_ctx_cmps), 630 MRS_FIELD(ID_AA64DFR0, WRPs, false, MRS_LOWER, MRS_USERSPACE, 631 id_aa64dfr0_wrps), 632 MRS_FIELD(ID_AA64DFR0, PMSS, false, MRS_LOWER, 0, id_aa64dfr0_pmss), 633 MRS_FIELD(ID_AA64DFR0, BRPs, false, MRS_LOWER, MRS_USERSPACE, 634 id_aa64dfr0_brps), 635 MRS_FIELD(ID_AA64DFR0, PMUVer, false, MRS_LOWER, 0, id_aa64dfr0_pmuver), 636 MRS_FIELD(ID_AA64DFR0, TraceVer, false, MRS_LOWER, 0, 637 id_aa64dfr0_tracever), 638 MRS_FIELD(ID_AA64DFR0, DebugVer, false, MRS_LOWER | MRS_SAFE(0x6), 0, 639 id_aa64dfr0_debugver), 640 MRS_FIELD_END, 641 }; 642 643 644 /* ID_AA64DFR1_EL1 */ 645 static const struct mrs_field_value id_aa64dfr1_dpfzs[] = { 646 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR1, DPFZS, NONE, IMPL), 647 MRS_FIELD_VALUE_END, 648 }; 649 650 static const struct mrs_field_value id_aa64dfr1_pmicntr[] = { 651 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR1, PMICNTR, NONE, IMPL), 652 MRS_FIELD_VALUE_END, 653 }; 654 655 static const struct mrs_field_value id_aa64dfr1_spmu[] = { 656 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64DFR1, SPMU, NONE, IMPL), 657 MRS_FIELD_VALUE_END, 658 }; 659 660 static const struct mrs_field id_aa64dfr1_fields[] = { 661 MRS_FIELD(ID_AA64DFR1, DPFZS, false, MRS_LOWER, 0, id_aa64dfr1_dpfzs), 662 MRS_FIELD(ID_AA64DFR1, PMICNTR, false, MRS_LOWER, 0, id_aa64dfr1_pmicntr), 663 MRS_FIELD(ID_AA64DFR1, SPMU, false, MRS_LOWER, 0, id_aa64dfr1_spmu), 664 MRS_FIELD_END, 665 }; 666 667 668 /* ID_AA64ISAR0_EL1 */ 669 static const struct mrs_field_value id_aa64isar0_rndr[] = { 670 MRS_FIELD_VALUE(ID_AA64ISAR0_RNDR_NONE, ""), 671 MRS_FIELD_VALUE(ID_AA64ISAR0_RNDR_IMPL, "RNG"), 672 MRS_FIELD_VALUE_END, 673 }; 674 675 static const struct mrs_field_hwcap id_aa64isar0_rndr_caps[] = { 676 MRS_HWCAP(2, HWCAP2_RNG, ID_AA64ISAR0_RNDR_IMPL), 677 MRS_HWCAP_END 678 }; 679 680 static const struct mrs_field_value id_aa64isar0_tlb[] = { 681 MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_NONE, ""), 682 MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_TLBIOS, "TLBI-OS"), 683 MRS_FIELD_VALUE(ID_AA64ISAR0_TLB_TLBIOSR, "TLBI-OSR"), 684 MRS_FIELD_VALUE_END, 685 }; 686 687 static const struct mrs_field_value id_aa64isar0_ts[] = { 688 MRS_FIELD_VALUE(ID_AA64ISAR0_TS_NONE, ""), 689 MRS_FIELD_VALUE(ID_AA64ISAR0_TS_CondM_8_4, "CondM-8.4"), 690 MRS_FIELD_VALUE(ID_AA64ISAR0_TS_CondM_8_5, "CondM-8.5"), 691 MRS_FIELD_VALUE_END, 692 }; 693 694 static const struct mrs_field_hwcap id_aa64isar0_ts_caps[] = { 695 MRS_HWCAP(1, HWCAP_FLAGM, ID_AA64ISAR0_TS_CondM_8_4), 696 MRS_HWCAP(2, HWCAP2_FLAGM2, ID_AA64ISAR0_TS_CondM_8_5), 697 MRS_HWCAP_END 698 }; 699 700 static const struct mrs_field_value id_aa64isar0_fhm[] = { 701 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, FHM, NONE, IMPL), 702 MRS_FIELD_VALUE_END, 703 }; 704 705 static const struct mrs_field_hwcap id_aa64isar0_fhm_caps[] = { 706 MRS_HWCAP(1, HWCAP_ASIMDFHM, ID_AA64ISAR0_FHM_IMPL), 707 MRS_HWCAP_END 708 }; 709 710 static const struct mrs_field_value id_aa64isar0_dp[] = { 711 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, DP, NONE, IMPL), 712 MRS_FIELD_VALUE_END, 713 }; 714 715 static const struct mrs_field_hwcap id_aa64isar0_dp_caps[] = { 716 MRS_HWCAP(1, HWCAP_ASIMDDP, ID_AA64ISAR0_DP_IMPL), 717 MRS_HWCAP_END 718 }; 719 720 static const struct mrs_field_value id_aa64isar0_sm4[] = { 721 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SM4, NONE, IMPL), 722 MRS_FIELD_VALUE_END, 723 }; 724 725 static const struct mrs_field_hwcap id_aa64isar0_sm4_caps[] = { 726 MRS_HWCAP(1, HWCAP_SM4, ID_AA64ISAR0_SM4_IMPL), 727 MRS_HWCAP_END 728 }; 729 730 static const struct mrs_field_value id_aa64isar0_sm3[] = { 731 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SM3, NONE, IMPL), 732 MRS_FIELD_VALUE_END, 733 }; 734 735 static const struct mrs_field_hwcap id_aa64isar0_sm3_caps[] = { 736 MRS_HWCAP(1, HWCAP_SM3, ID_AA64ISAR0_SM3_IMPL), 737 MRS_HWCAP_END 738 }; 739 740 static const struct mrs_field_value id_aa64isar0_sha3[] = { 741 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA3, NONE, IMPL), 742 MRS_FIELD_VALUE_END, 743 }; 744 745 static const struct mrs_field_hwcap id_aa64isar0_sha3_caps[] = { 746 MRS_HWCAP(1, HWCAP_SHA3, ID_AA64ISAR0_SHA3_IMPL), 747 MRS_HWCAP_END 748 }; 749 750 static const struct mrs_field_value id_aa64isar0_rdm[] = { 751 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, RDM, NONE, IMPL), 752 MRS_FIELD_VALUE_END, 753 }; 754 755 static const struct mrs_field_hwcap id_aa64isar0_rdm_caps[] = { 756 MRS_HWCAP(1, HWCAP_ASIMDRDM, ID_AA64ISAR0_RDM_IMPL), 757 MRS_HWCAP_END 758 }; 759 760 static const struct mrs_field_value id_aa64isar0_tme[] = { 761 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, TME, NONE, IMPL), 762 MRS_FIELD_VALUE_END, 763 }; 764 765 static const struct mrs_field_value id_aa64isar0_atomic[] = { 766 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, Atomic, NONE, IMPL), 767 MRS_FIELD_VALUE_END, 768 }; 769 770 static const struct mrs_field_hwcap id_aa64isar0_atomic_caps[] = { 771 MRS_HWCAP(1, HWCAP_ATOMICS, ID_AA64ISAR0_Atomic_IMPL), 772 MRS_HWCAP_END 773 }; 774 775 static const struct mrs_field_value id_aa64isar0_crc32[] = { 776 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, CRC32, NONE, BASE), 777 MRS_FIELD_VALUE_END, 778 }; 779 780 static const struct mrs_field_hwcap id_aa64isar0_crc32_caps[] = { 781 MRS_HWCAP(1, HWCAP_CRC32, ID_AA64ISAR0_CRC32_BASE), 782 MRS_HWCAP_END 783 }; 784 785 static const struct mrs_field_value id_aa64isar0_sha2[] = { 786 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA2, NONE, BASE), 787 MRS_FIELD_VALUE(ID_AA64ISAR0_SHA2_512, "SHA2+SHA512"), 788 MRS_FIELD_VALUE_END, 789 }; 790 791 static const struct mrs_field_hwcap id_aa64isar0_sha2_caps[] = { 792 MRS_HWCAP(1, HWCAP_SHA2, ID_AA64ISAR0_SHA2_BASE), 793 MRS_HWCAP(1, HWCAP_SHA512, ID_AA64ISAR0_SHA2_512), 794 MRS_HWCAP_END 795 }; 796 797 static const struct mrs_field_value id_aa64isar0_sha1[] = { 798 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, SHA1, NONE, BASE), 799 MRS_FIELD_VALUE_END, 800 }; 801 802 static const struct mrs_field_hwcap id_aa64isar0_sha1_caps[] = { 803 MRS_HWCAP(1, HWCAP_SHA1, ID_AA64ISAR0_SHA1_BASE), 804 MRS_HWCAP_END 805 }; 806 807 static const struct mrs_field_value id_aa64isar0_aes[] = { 808 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR0, AES, NONE, BASE), 809 MRS_FIELD_VALUE(ID_AA64ISAR0_AES_PMULL, "AES+PMULL"), 810 MRS_FIELD_VALUE_END, 811 }; 812 813 static const struct mrs_field_hwcap id_aa64isar0_aes_caps[] = { 814 MRS_HWCAP(1, HWCAP_AES, ID_AA64ISAR0_AES_BASE), 815 MRS_HWCAP(1, HWCAP_PMULL, ID_AA64ISAR0_AES_PMULL), 816 MRS_HWCAP_END 817 }; 818 819 static const struct mrs_field id_aa64isar0_fields[] = { 820 MRS_FIELD_HWCAP(ID_AA64ISAR0, RNDR, false, MRS_LOWER, MRS_USERSPACE, 821 id_aa64isar0_rndr, id_aa64isar0_rndr_caps), 822 MRS_FIELD(ID_AA64ISAR0, TLB, false, MRS_LOWER, 0, id_aa64isar0_tlb), 823 MRS_FIELD_HWCAP(ID_AA64ISAR0, TS, false, MRS_LOWER, MRS_USERSPACE, 824 id_aa64isar0_ts, id_aa64isar0_ts_caps), 825 MRS_FIELD_HWCAP(ID_AA64ISAR0, FHM, false, MRS_LOWER, MRS_USERSPACE, 826 id_aa64isar0_fhm, id_aa64isar0_fhm_caps), 827 MRS_FIELD_HWCAP(ID_AA64ISAR0, DP, false, MRS_LOWER, MRS_USERSPACE, 828 id_aa64isar0_dp, id_aa64isar0_dp_caps), 829 MRS_FIELD_HWCAP(ID_AA64ISAR0, SM4, false, MRS_LOWER, MRS_USERSPACE, 830 id_aa64isar0_sm4, id_aa64isar0_sm4_caps), 831 MRS_FIELD_HWCAP(ID_AA64ISAR0, SM3, false, MRS_LOWER, MRS_USERSPACE, 832 id_aa64isar0_sm3, id_aa64isar0_sm3_caps), 833 MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA3, false, MRS_LOWER, MRS_USERSPACE, 834 id_aa64isar0_sha3, id_aa64isar0_sha3_caps), 835 MRS_FIELD_HWCAP(ID_AA64ISAR0, RDM, false, MRS_LOWER, MRS_USERSPACE, 836 id_aa64isar0_rdm, id_aa64isar0_rdm_caps), 837 MRS_FIELD(ID_AA64ISAR0, TME, false, MRS_LOWER, 0, id_aa64isar0_tme), 838 MRS_FIELD_HWCAP(ID_AA64ISAR0, Atomic, false, MRS_LOWER, MRS_USERSPACE, 839 id_aa64isar0_atomic, id_aa64isar0_atomic_caps), 840 MRS_FIELD_HWCAP(ID_AA64ISAR0, CRC32, false, MRS_LOWER, MRS_USERSPACE, 841 id_aa64isar0_crc32, id_aa64isar0_crc32_caps), 842 MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA2, false, MRS_LOWER, MRS_USERSPACE, 843 id_aa64isar0_sha2, id_aa64isar0_sha2_caps), 844 MRS_FIELD_HWCAP(ID_AA64ISAR0, SHA1, false, MRS_LOWER, MRS_USERSPACE, 845 id_aa64isar0_sha1, id_aa64isar0_sha1_caps), 846 MRS_FIELD_HWCAP(ID_AA64ISAR0, AES, false, MRS_LOWER, MRS_USERSPACE, 847 id_aa64isar0_aes, id_aa64isar0_aes_caps), 848 MRS_FIELD_END, 849 }; 850 851 852 /* ID_AA64ISAR1_EL1 */ 853 static const struct mrs_field_value id_aa64isar1_ls64[] = { 854 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, LS64, NONE, IMPL), 855 MRS_FIELD_VALUE(ID_AA64ISAR1_LS64_V, "LS64v"), 856 MRS_FIELD_VALUE(ID_AA64ISAR1_LS64_ACCDATA, "LS64+ACCDATA"), 857 MRS_FIELD_VALUE_END, 858 }; 859 860 static const struct mrs_field_value id_aa64isar1_xs[] = { 861 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, XS, NONE, IMPL), 862 MRS_FIELD_VALUE_END, 863 }; 864 865 static const struct mrs_field_value id_aa64isar1_i8mm[] = { 866 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, I8MM, NONE, IMPL), 867 MRS_FIELD_VALUE_END, 868 }; 869 870 static const struct mrs_field_hwcap id_aa64isar1_i8mm_caps[] = { 871 MRS_HWCAP(2, HWCAP2_I8MM, ID_AA64ISAR1_I8MM_IMPL), 872 MRS_HWCAP_END 873 }; 874 875 static const struct mrs_field_value id_aa64isar1_dgh[] = { 876 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, DGH, NONE, IMPL), 877 MRS_FIELD_VALUE_END, 878 }; 879 880 static const struct mrs_field_hwcap id_aa64isar1_dgh_caps[] = { 881 MRS_HWCAP(2, HWCAP2_DGH, ID_AA64ISAR1_DGH_IMPL), 882 MRS_HWCAP_END 883 }; 884 885 static const struct mrs_field_value id_aa64isar1_bf16[] = { 886 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, BF16, NONE, IMPL), 887 MRS_FIELD_VALUE(ID_AA64ISAR1_BF16_EBF, "EBF16"), 888 MRS_FIELD_VALUE_END, 889 }; 890 891 static const struct mrs_field_hwcap id_aa64isar1_bf16_caps[] = { 892 MRS_HWCAP(2, HWCAP2_BF16, ID_AA64ISAR1_BF16_IMPL), 893 MRS_HWCAP_END 894 }; 895 896 static const struct mrs_field_value id_aa64isar1_specres[] = { 897 MRS_FIELD_VALUE(ID_AA64ISAR1_SPECRES_NONE, ""), 898 MRS_FIELD_VALUE(ID_AA64ISAR1_SPECRES_8_5, "PredInv v8.5"), 899 MRS_FIELD_VALUE(ID_AA64ISAR1_SPECRES_8_9, "PredInv v8.9"), 900 MRS_FIELD_VALUE_END, 901 }; 902 903 static const struct mrs_field_value id_aa64isar1_sb[] = { 904 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, SB, NONE, IMPL), 905 MRS_FIELD_VALUE_END, 906 }; 907 908 static const struct mrs_field_hwcap id_aa64isar1_sb_caps[] = { 909 MRS_HWCAP(1, HWCAP_SB, ID_AA64ISAR1_SB_IMPL), 910 MRS_HWCAP_END 911 }; 912 913 static const struct mrs_field_value id_aa64isar1_frintts[] = { 914 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, FRINTTS, NONE, IMPL), 915 MRS_FIELD_VALUE_END, 916 }; 917 918 static const struct mrs_field_hwcap id_aa64isar1_frintts_caps[] = { 919 MRS_HWCAP(2, HWCAP2_FRINT, ID_AA64ISAR1_FRINTTS_IMPL), 920 MRS_HWCAP_END 921 }; 922 923 static const struct mrs_field_value id_aa64isar1_gpi[] = { 924 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, GPI, NONE, IMPL), 925 MRS_FIELD_VALUE_END, 926 }; 927 928 static const struct mrs_field_hwcap id_aa64isar1_gpi_caps[] = { 929 MRS_HWCAP(1, HWCAP_PACG, ID_AA64ISAR1_GPI_IMPL), 930 MRS_HWCAP_END 931 }; 932 933 static const struct mrs_field_value id_aa64isar1_gpa[] = { 934 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, GPA, NONE, IMPL), 935 MRS_FIELD_VALUE_END, 936 }; 937 938 static const struct mrs_field_hwcap id_aa64isar1_gpa_caps[] = { 939 MRS_HWCAP(1, HWCAP_PACG, ID_AA64ISAR1_GPA_IMPL), 940 MRS_HWCAP_END 941 }; 942 943 static const struct mrs_field_value id_aa64isar1_lrcpc[] = { 944 MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_NONE, ""), 945 MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_RCPC_8_3, "RCPC-8.3"), 946 MRS_FIELD_VALUE(ID_AA64ISAR1_LRCPC_RCPC_8_4, "RCPC-8.4"), 947 MRS_FIELD_VALUE_END, 948 }; 949 950 static const struct mrs_field_hwcap id_aa64isar1_lrcpc_caps[] = { 951 MRS_HWCAP(1, HWCAP_LRCPC, ID_AA64ISAR1_LRCPC_RCPC_8_3), 952 MRS_HWCAP(1, HWCAP_ILRCPC, ID_AA64ISAR1_LRCPC_RCPC_8_4), 953 MRS_HWCAP_END 954 }; 955 956 static const struct mrs_field_value id_aa64isar1_fcma[] = { 957 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, FCMA, NONE, IMPL), 958 MRS_FIELD_VALUE_END, 959 }; 960 961 static const struct mrs_field_hwcap id_aa64isar1_fcma_caps[] = { 962 MRS_HWCAP(1, HWCAP_FCMA, ID_AA64ISAR1_FCMA_IMPL), 963 MRS_HWCAP_END 964 }; 965 966 static const struct mrs_field_value id_aa64isar1_jscvt[] = { 967 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR1, JSCVT, NONE, IMPL), 968 MRS_FIELD_VALUE_END, 969 }; 970 971 static const struct mrs_field_hwcap id_aa64isar1_jscvt_caps[] = { 972 MRS_HWCAP(1, HWCAP_JSCVT, ID_AA64ISAR1_JSCVT_IMPL), 973 MRS_HWCAP_END 974 }; 975 976 static const struct mrs_field_value id_aa64isar1_api[] = { 977 MRS_FIELD_VALUE(ID_AA64ISAR1_API_NONE, ""), 978 MRS_FIELD_VALUE(ID_AA64ISAR1_API_PAC, "API PAC"), 979 MRS_FIELD_VALUE(ID_AA64ISAR1_API_EPAC, "API EPAC"), 980 MRS_FIELD_VALUE(ID_AA64ISAR1_API_EPAC2, "Impl PAuth+EPAC2"), 981 MRS_FIELD_VALUE(ID_AA64ISAR1_API_FPAC, "Impl PAuth+FPAC"), 982 MRS_FIELD_VALUE(ID_AA64ISAR1_API_FPAC_COMBINED, 983 "Impl PAuth+FPAC+Combined"), 984 MRS_FIELD_VALUE_END, 985 }; 986 987 static const struct mrs_field_hwcap id_aa64isar1_api_caps[] = { 988 MRS_HWCAP(1, HWCAP_PACA, ID_AA64ISAR1_API_PAC), 989 MRS_HWCAP_END 990 }; 991 992 static const struct mrs_field_value id_aa64isar1_apa[] = { 993 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_NONE, ""), 994 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_PAC, "APA PAC"), 995 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_EPAC, "APA EPAC"), 996 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_EPAC2, "APA EPAC2"), 997 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_FPAC, "APA FPAC"), 998 MRS_FIELD_VALUE(ID_AA64ISAR1_APA_FPAC_COMBINED, 999 "APA FPAC+Combined"), 1000 MRS_FIELD_VALUE_END, 1001 }; 1002 1003 static const struct mrs_field_hwcap id_aa64isar1_apa_caps[] = { 1004 MRS_HWCAP(1, HWCAP_PACA, ID_AA64ISAR1_APA_PAC), 1005 MRS_HWCAP_END 1006 }; 1007 1008 static const struct mrs_field_value id_aa64isar1_dpb[] = { 1009 MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_NONE, ""), 1010 MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_DCCVAP, "DCPoP"), 1011 MRS_FIELD_VALUE(ID_AA64ISAR1_DPB_DCCVADP, "DCCVADP"), 1012 MRS_FIELD_VALUE_END, 1013 }; 1014 1015 static const struct mrs_field_hwcap id_aa64isar1_dpb_caps[] = { 1016 MRS_HWCAP(1, HWCAP_DCPOP, ID_AA64ISAR1_DPB_DCCVAP), 1017 MRS_HWCAP(2, HWCAP2_DCPODP, ID_AA64ISAR1_DPB_DCCVADP), 1018 MRS_HWCAP_END 1019 }; 1020 1021 static const struct mrs_field id_aa64isar1_fields[] = { 1022 MRS_FIELD(ID_AA64ISAR1, LS64, false, MRS_LOWER, 0, id_aa64isar1_ls64), 1023 MRS_FIELD(ID_AA64ISAR1, XS, false, MRS_LOWER, 0, id_aa64isar1_xs), 1024 MRS_FIELD_HWCAP(ID_AA64ISAR1, I8MM, false, MRS_LOWER, MRS_USERSPACE, 1025 id_aa64isar1_i8mm, id_aa64isar1_i8mm_caps), 1026 MRS_FIELD_HWCAP(ID_AA64ISAR1, DGH, false, MRS_LOWER, MRS_USERSPACE, 1027 id_aa64isar1_dgh, id_aa64isar1_dgh_caps), 1028 MRS_FIELD_HWCAP(ID_AA64ISAR1, BF16, false, MRS_LOWER, MRS_USERSPACE, 1029 id_aa64isar1_bf16, id_aa64isar1_bf16_caps), 1030 MRS_FIELD(ID_AA64ISAR1, SPECRES, false, MRS_LOWER, 0, 1031 id_aa64isar1_specres), 1032 MRS_FIELD_HWCAP(ID_AA64ISAR1, SB, false, MRS_LOWER, MRS_USERSPACE, 1033 id_aa64isar1_sb, id_aa64isar1_sb_caps), 1034 MRS_FIELD_HWCAP(ID_AA64ISAR1, FRINTTS, false, MRS_LOWER, MRS_USERSPACE, 1035 id_aa64isar1_frintts, id_aa64isar1_frintts_caps), 1036 MRS_FIELD_HWCAP(ID_AA64ISAR1, GPI, false, MRS_LOWER, MRS_USERSPACE, 1037 id_aa64isar1_gpi, id_aa64isar1_gpi_caps), 1038 MRS_FIELD_HWCAP(ID_AA64ISAR1, GPA, false, MRS_LOWER, MRS_USERSPACE, 1039 id_aa64isar1_gpa, id_aa64isar1_gpa_caps), 1040 MRS_FIELD_HWCAP(ID_AA64ISAR1, LRCPC, false, MRS_LOWER, MRS_USERSPACE, 1041 id_aa64isar1_lrcpc, id_aa64isar1_lrcpc_caps), 1042 MRS_FIELD_HWCAP(ID_AA64ISAR1, FCMA, false, MRS_LOWER, MRS_USERSPACE, 1043 id_aa64isar1_fcma, id_aa64isar1_fcma_caps), 1044 MRS_FIELD_HWCAP(ID_AA64ISAR1, JSCVT, false, MRS_LOWER, MRS_USERSPACE, 1045 id_aa64isar1_jscvt, id_aa64isar1_jscvt_caps), 1046 MRS_FIELD_HWCAP(ID_AA64ISAR1, API, false, MRS_LOWER, MRS_USERSPACE, 1047 id_aa64isar1_api, id_aa64isar1_api_caps), 1048 MRS_FIELD_HWCAP(ID_AA64ISAR1, APA, false, MRS_LOWER, MRS_USERSPACE, 1049 id_aa64isar1_apa, id_aa64isar1_apa_caps), 1050 MRS_FIELD_HWCAP(ID_AA64ISAR1, DPB, false, MRS_LOWER, MRS_USERSPACE, 1051 id_aa64isar1_dpb, id_aa64isar1_dpb_caps), 1052 MRS_FIELD_END, 1053 }; 1054 1055 1056 /* ID_AA64ISAR2_EL1 */ 1057 static const struct mrs_field_value id_aa64isar2_ats1a[] = { 1058 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, ATS1A, NONE, IMPL), 1059 MRS_FIELD_VALUE_END, 1060 }; 1061 1062 static const struct mrs_field_value id_aa64isar2_cssc[] = { 1063 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, CSSC, NONE, IMPL), 1064 MRS_FIELD_VALUE_END, 1065 }; 1066 1067 static const struct mrs_field_value id_aa64isar2_rprfm[] = { 1068 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, RPRFM, NONE, IMPL), 1069 MRS_FIELD_VALUE_END, 1070 }; 1071 1072 static const struct mrs_field_value id_aa64isar2_prfmslc[] = { 1073 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, PRFMSLC, NONE, IMPL), 1074 MRS_FIELD_VALUE_END, 1075 }; 1076 1077 static const struct mrs_field_value id_aa64isar2_clrbhb[] = { 1078 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, CLRBHB, NONE, IMPL), 1079 MRS_FIELD_VALUE_END, 1080 }; 1081 1082 static const struct mrs_field_value id_aa64isar2_pac_frac[] = { 1083 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, PAC_frac, NONE, IMPL), 1084 MRS_FIELD_VALUE_END, 1085 }; 1086 1087 static const struct mrs_field_value id_aa64isar2_bc[] = { 1088 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, BC, NONE, IMPL), 1089 MRS_FIELD_VALUE_END, 1090 }; 1091 1092 static const struct mrs_field_value id_aa64isar2_mops[] = { 1093 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, MOPS, NONE, IMPL), 1094 MRS_FIELD_VALUE_END, 1095 }; 1096 1097 static const struct mrs_field_hwcap id_aa64isar2_mops_caps[] = { 1098 MRS_HWCAP(2, HWCAP2_MOPS, ID_AA64ISAR2_MOPS_IMPL), 1099 MRS_HWCAP_END 1100 }; 1101 1102 static const struct mrs_field_value id_aa64isar2_apa3[] = { 1103 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_NONE, ""), 1104 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_PAC, "APA3 PAC"), 1105 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_EPAC, "APA3 EPAC"), 1106 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_EPAC2, "APA3 EPAC2"), 1107 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_FPAC, "APA3 FPAC"), 1108 MRS_FIELD_VALUE(ID_AA64ISAR2_APA3_FPAC_COMBINED, 1109 "APA3 FPAC+Combined"), 1110 MRS_FIELD_VALUE_END, 1111 }; 1112 1113 static const struct mrs_field_hwcap id_aa64isar2_apa3_caps[] = { 1114 MRS_HWCAP(1, HWCAP_PACA, ID_AA64ISAR2_APA3_PAC), 1115 MRS_HWCAP_END 1116 }; 1117 1118 static const struct mrs_field_value id_aa64isar2_gpa3[] = { 1119 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, GPA3, NONE, IMPL), 1120 MRS_FIELD_VALUE_END, 1121 }; 1122 1123 static const struct mrs_field_hwcap id_aa64isar2_gpa3_caps[] = { 1124 MRS_HWCAP(1, HWCAP_PACG, ID_AA64ISAR2_GPA3_IMPL), 1125 MRS_HWCAP_END 1126 }; 1127 1128 static const struct mrs_field_value id_aa64isar2_rpres[] = { 1129 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, RPRES, NONE, IMPL), 1130 MRS_FIELD_VALUE_END, 1131 }; 1132 1133 static const struct mrs_field_hwcap id_aa64isar2_rpres_caps[] = { 1134 MRS_HWCAP(2, HWCAP2_RPRES, ID_AA64ISAR2_RPRES_IMPL), 1135 MRS_HWCAP_END 1136 }; 1137 1138 static const struct mrs_field_value id_aa64isar2_wfxt[] = { 1139 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ISAR2, WFxT, NONE, IMPL), 1140 MRS_FIELD_VALUE_END, 1141 }; 1142 1143 static const struct mrs_field_hwcap id_aa64isar2_wfxt_caps[] = { 1144 MRS_HWCAP(2, HWCAP2_WFXT, ID_AA64ISAR2_WFxT_IMPL), 1145 MRS_HWCAP_END 1146 }; 1147 1148 static const struct mrs_field id_aa64isar2_fields[] = { 1149 MRS_FIELD(ID_AA64ISAR2, ATS1A, false, MRS_LOWER, 0, id_aa64isar2_ats1a), 1150 MRS_FIELD(ID_AA64ISAR2, CSSC, false, MRS_LOWER, 0, id_aa64isar2_cssc), 1151 MRS_FIELD(ID_AA64ISAR2, RPRFM, false, MRS_LOWER, 0, id_aa64isar2_rprfm), 1152 MRS_FIELD(ID_AA64ISAR2, PRFMSLC, false, MRS_LOWER, 0, id_aa64isar2_prfmslc), 1153 MRS_FIELD(ID_AA64ISAR2, CLRBHB, false, MRS_LOWER, 0, id_aa64isar2_clrbhb), 1154 MRS_FIELD(ID_AA64ISAR2, PAC_frac, false, MRS_LOWER, 0, 1155 id_aa64isar2_pac_frac), 1156 MRS_FIELD(ID_AA64ISAR2, BC, false, MRS_LOWER, 0, id_aa64isar2_bc), 1157 MRS_FIELD_HWCAP(ID_AA64ISAR2, MOPS, false, MRS_LOWER, MRS_USERSPACE, 1158 id_aa64isar2_mops, id_aa64isar2_mops_caps), 1159 MRS_FIELD_HWCAP(ID_AA64ISAR2, APA3, false, MRS_LOWER, MRS_USERSPACE, 1160 id_aa64isar2_apa3, id_aa64isar2_apa3_caps), 1161 MRS_FIELD_HWCAP(ID_AA64ISAR2, GPA3, false, MRS_LOWER, MRS_USERSPACE, 1162 id_aa64isar2_gpa3, id_aa64isar2_gpa3_caps), 1163 MRS_FIELD_HWCAP(ID_AA64ISAR2, RPRES, false, MRS_LOWER, MRS_USERSPACE, 1164 id_aa64isar2_rpres, id_aa64isar2_rpres_caps), 1165 MRS_FIELD_HWCAP(ID_AA64ISAR2, WFxT, false, MRS_LOWER, 0, 1166 id_aa64isar2_wfxt, id_aa64isar2_wfxt_caps), 1167 MRS_FIELD_END, 1168 }; 1169 1170 1171 /* ID_AA64MMFR0_EL1 */ 1172 static const struct mrs_field_value id_aa64mmfr0_ecv[] = { 1173 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, ECV, NONE, IMPL), 1174 MRS_FIELD_VALUE(ID_AA64MMFR0_ECV_POFF, "ECV POFF"), 1175 MRS_FIELD_VALUE_END, 1176 }; 1177 1178 static const struct mrs_field_value id_aa64mmfr0_fgt[] = { 1179 MRS_FIELD_VALUE(ID_AA64MMFR0_FGT_NONE, ""), 1180 MRS_FIELD_VALUE(ID_AA64MMFR0_FGT_8_6, "FGT v8.6"), 1181 MRS_FIELD_VALUE(ID_AA64MMFR0_FGT_8_9, "FGT v8.9"), 1182 MRS_FIELD_VALUE_END, 1183 }; 1184 1185 static const struct mrs_field_value id_aa64mmfr0_exs[] = { 1186 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, ExS, ALL, IMPL), 1187 MRS_FIELD_VALUE_END, 1188 }; 1189 1190 static const struct mrs_field_value id_aa64mmfr0_tgran4_2[] = { 1191 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_TGran4, ""), 1192 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_NONE, "No S2 TGran4"), 1193 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_IMPL, "S2 TGran4"), 1194 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_2_LPA2, "S2 TGran4+LPA2"), 1195 MRS_FIELD_VALUE_END, 1196 }; 1197 1198 static const struct mrs_field_value id_aa64mmfr0_tgran64_2[] = { 1199 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_TGran64, ""), 1200 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_NONE, "No S2 TGran64"), 1201 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran64_2_IMPL, "S2 TGran64"), 1202 MRS_FIELD_VALUE_END, 1203 }; 1204 1205 static const struct mrs_field_value id_aa64mmfr0_tgran16_2[] = { 1206 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_TGran16, ""), 1207 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_NONE, "No S2 TGran16"), 1208 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_IMPL, "S2 TGran16"), 1209 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_2_LPA2, "S2 TGran16+LPA2"), 1210 MRS_FIELD_VALUE_END, 1211 }; 1212 1213 static const struct mrs_field_value id_aa64mmfr0_tgran4[] = { 1214 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran4, NONE, IMPL), 1215 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran4_LPA2, "TGran4+LPA2"), 1216 MRS_FIELD_VALUE_END, 1217 }; 1218 1219 static const struct mrs_field_value id_aa64mmfr0_tgran64[] = { 1220 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran64, NONE, IMPL), 1221 MRS_FIELD_VALUE_END, 1222 }; 1223 1224 static const struct mrs_field_value id_aa64mmfr0_tgran16[] = { 1225 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, TGran16, NONE, IMPL), 1226 MRS_FIELD_VALUE(ID_AA64MMFR0_TGran16_LPA2, "TGran16+LPA2"), 1227 MRS_FIELD_VALUE_END, 1228 }; 1229 1230 static const struct mrs_field_value id_aa64mmfr0_bigendel0[] = { 1231 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, BigEndEL0, FIXED, MIXED), 1232 MRS_FIELD_VALUE_END, 1233 }; 1234 1235 static const struct mrs_field_value id_aa64mmfr0_snsmem[] = { 1236 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, SNSMem, NONE, DISTINCT), 1237 MRS_FIELD_VALUE_END, 1238 }; 1239 1240 static const struct mrs_field_value id_aa64mmfr0_bigend[] = { 1241 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR0, BigEnd, FIXED, MIXED), 1242 MRS_FIELD_VALUE_END, 1243 }; 1244 1245 static const struct mrs_field_value id_aa64mmfr0_asidbits[] = { 1246 MRS_FIELD_VALUE(ID_AA64MMFR0_ASIDBits_8, "8bit ASID"), 1247 MRS_FIELD_VALUE(ID_AA64MMFR0_ASIDBits_16, "16bit ASID"), 1248 MRS_FIELD_VALUE_END, 1249 }; 1250 1251 static const struct mrs_field_value id_aa64mmfr0_parange[] = { 1252 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4G, "4GB PA"), 1253 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_64G, "64GB PA"), 1254 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_1T, "1TB PA"), 1255 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4T, "4TB PA"), 1256 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_16T, "16TB PA"), 1257 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_256T, "256TB PA"), 1258 MRS_FIELD_VALUE(ID_AA64MMFR0_PARange_4P, "4PB PA"), 1259 MRS_FIELD_VALUE_END, 1260 }; 1261 1262 static const struct mrs_field id_aa64mmfr0_fields[] = { 1263 MRS_FIELD(ID_AA64MMFR0, ECV, false, MRS_LOWER, 0, id_aa64mmfr0_ecv), 1264 MRS_FIELD(ID_AA64MMFR0, FGT, false, MRS_LOWER, 0, id_aa64mmfr0_fgt), 1265 MRS_FIELD(ID_AA64MMFR0, ExS, false, MRS_LOWER, 0, id_aa64mmfr0_exs), 1266 MRS_FIELD(ID_AA64MMFR0, TGran4_2, false, MRS_LOWER, 0, 1267 id_aa64mmfr0_tgran4_2), 1268 MRS_FIELD(ID_AA64MMFR0, TGran64_2, false, MRS_LOWER, 0, 1269 id_aa64mmfr0_tgran64_2), 1270 MRS_FIELD(ID_AA64MMFR0, TGran16_2, false, MRS_LOWER, 0, 1271 id_aa64mmfr0_tgran16_2), 1272 MRS_FIELD(ID_AA64MMFR0, TGran4, true, MRS_LOWER, 0, 1273 id_aa64mmfr0_tgran4), 1274 MRS_FIELD(ID_AA64MMFR0, TGran64, true, MRS_LOWER, 0, 1275 id_aa64mmfr0_tgran64), 1276 MRS_FIELD(ID_AA64MMFR0, TGran16, false, MRS_LOWER, 0, 1277 id_aa64mmfr0_tgran16), 1278 MRS_FIELD(ID_AA64MMFR0, BigEndEL0, false, MRS_LOWER, 0, 1279 id_aa64mmfr0_bigendel0), 1280 MRS_FIELD(ID_AA64MMFR0, SNSMem, false, MRS_LOWER, 0, 1281 id_aa64mmfr0_snsmem), 1282 MRS_FIELD(ID_AA64MMFR0, BigEnd, false, MRS_LOWER, 0, 1283 id_aa64mmfr0_bigend), 1284 MRS_FIELD(ID_AA64MMFR0, ASIDBits, false, MRS_LOWER, 0, 1285 id_aa64mmfr0_asidbits), 1286 MRS_FIELD(ID_AA64MMFR0, PARange, false, MRS_LOWER, 0, 1287 id_aa64mmfr0_parange), 1288 MRS_FIELD_END, 1289 }; 1290 1291 1292 /* ID_AA64MMFR1_EL1 */ 1293 static const struct mrs_field_value id_aa64mmfr1_ecbhb[] = { 1294 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, ECBHB, NONE, IMPL), 1295 MRS_FIELD_VALUE_END, 1296 }; 1297 1298 static const struct mrs_field_value id_aa64mmfr1_cmovw[] = { 1299 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, CMOVW, NONE, IMPL), 1300 MRS_FIELD_VALUE_END, 1301 }; 1302 1303 static const struct mrs_field_value id_aa64mmfr1_tidcp1[] = { 1304 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, TIDCP1, NONE, IMPL), 1305 MRS_FIELD_VALUE_END, 1306 }; 1307 1308 static const struct mrs_field_value id_aa64mmfr1_ntlbpa[] = { 1309 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, nTLBPA, NONE, IMPL), 1310 MRS_FIELD_VALUE_END, 1311 }; 1312 1313 static const struct mrs_field_value id_aa64mmfr1_afp[] = { 1314 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, AFP, NONE, IMPL), 1315 MRS_FIELD_VALUE_END, 1316 }; 1317 1318 static const struct mrs_field_hwcap id_aa64mmfr1_afp_caps[] = { 1319 MRS_HWCAP(2, HWCAP2_AFP, ID_AA64MMFR1_AFP_IMPL), 1320 MRS_HWCAP_END 1321 }; 1322 1323 static const struct mrs_field_value id_aa64mmfr1_hcx[] = { 1324 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, HCX, NONE, IMPL), 1325 MRS_FIELD_VALUE_END, 1326 }; 1327 1328 static const struct mrs_field_value id_aa64mmfr1_ets[] = { 1329 MRS_FIELD_VALUE(ID_AA64MMFR1_ETS_NONE, ""), 1330 MRS_FIELD_VALUE(ID_AA64MMFR1_ETS_NONE2, ""), 1331 MRS_FIELD_VALUE(ID_AA64MMFR1_ETS_IMPL, "ETS2"), 1332 MRS_FIELD_VALUE_END, 1333 }; 1334 1335 static const struct mrs_field_value id_aa64mmfr1_twed[] = { 1336 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, TWED, NONE, IMPL), 1337 MRS_FIELD_VALUE_END, 1338 }; 1339 1340 static const struct mrs_field_value id_aa64mmfr1_xnx[] = { 1341 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, XNX, NONE, IMPL), 1342 MRS_FIELD_VALUE_END, 1343 }; 1344 1345 static const struct mrs_field_value id_aa64mmfr1_specsei[] = { 1346 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, SpecSEI, NONE, IMPL), 1347 MRS_FIELD_VALUE_END, 1348 }; 1349 1350 static const struct mrs_field_value id_aa64mmfr1_pan[] = { 1351 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, PAN, NONE, IMPL), 1352 MRS_FIELD_VALUE(ID_AA64MMFR1_PAN_ATS1E1, "PAN+ATS1E1"), 1353 MRS_FIELD_VALUE(ID_AA64MMFR1_PAN_EPAN, "EPAN"), 1354 MRS_FIELD_VALUE_END, 1355 }; 1356 1357 static const struct mrs_field_value id_aa64mmfr1_lo[] = { 1358 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, LO, NONE, IMPL), 1359 MRS_FIELD_VALUE_END, 1360 }; 1361 1362 static const struct mrs_field_value id_aa64mmfr1_hpds[] = { 1363 MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_NONE, ""), 1364 MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_HPD, "HPD"), 1365 MRS_FIELD_VALUE(ID_AA64MMFR1_HPDS_TTPBHA, "HPD+TTPBHA"), 1366 MRS_FIELD_VALUE_END, 1367 }; 1368 1369 static const struct mrs_field_value id_aa64mmfr1_vh[] = { 1370 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR1, VH, NONE, IMPL), 1371 MRS_FIELD_VALUE_END, 1372 }; 1373 1374 static const struct mrs_field_value id_aa64mmfr1_vmidbits[] = { 1375 MRS_FIELD_VALUE(ID_AA64MMFR1_VMIDBits_8, "8bit VMID"), 1376 MRS_FIELD_VALUE(ID_AA64MMFR1_VMIDBits_16, "16bit VMID"), 1377 MRS_FIELD_VALUE_END, 1378 }; 1379 1380 static const struct mrs_field_value id_aa64mmfr1_hafdbs[] = { 1381 MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_NONE, ""), 1382 MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_AF, "HAF"), 1383 MRS_FIELD_VALUE(ID_AA64MMFR1_HAFDBS_AF_DBS, "HAF+DS"), 1384 MRS_FIELD_VALUE_END, 1385 }; 1386 1387 static const struct mrs_field id_aa64mmfr1_fields[] = { 1388 MRS_FIELD(ID_AA64MMFR1, ECBHB, false, MRS_LOWER, 0, id_aa64mmfr1_ecbhb), 1389 MRS_FIELD(ID_AA64MMFR1, CMOVW, false, MRS_LOWER, 0, id_aa64mmfr1_cmovw), 1390 MRS_FIELD(ID_AA64MMFR1, TIDCP1, false, MRS_LOWER, 0, 1391 id_aa64mmfr1_tidcp1), 1392 MRS_FIELD(ID_AA64MMFR1, nTLBPA, false, MRS_LOWER, 0, 1393 id_aa64mmfr1_ntlbpa), 1394 MRS_FIELD_HWCAP(ID_AA64MMFR1, AFP, false, MRS_LOWER, 0, 1395 id_aa64mmfr1_afp, id_aa64mmfr1_afp_caps), 1396 MRS_FIELD(ID_AA64MMFR1, HCX, false, MRS_LOWER, 0, id_aa64mmfr1_hcx), 1397 MRS_FIELD(ID_AA64MMFR1, ETS, false, MRS_LOWER, 0, id_aa64mmfr1_ets), 1398 MRS_FIELD(ID_AA64MMFR1, TWED, false, MRS_LOWER, 0, id_aa64mmfr1_twed), 1399 MRS_FIELD(ID_AA64MMFR1, XNX, false, MRS_LOWER, 0, id_aa64mmfr1_xnx), 1400 /* 1401 * SpecSEI != 0 indicates the CPU might generate an external abort 1402 * under speculation, while 0 indicates it can't happen. It's safer 1403 * to incorrectly indicate it might happen when it can't rather than 1404 * say it can't happen when it could. As such use the largest value 1405 * found in the system. 1406 */ 1407 MRS_FIELD(ID_AA64MMFR1, SpecSEI, false, MRS_HIGHER, 0, 1408 id_aa64mmfr1_specsei), 1409 MRS_FIELD(ID_AA64MMFR1, PAN, false, MRS_LOWER, 0, id_aa64mmfr1_pan), 1410 MRS_FIELD(ID_AA64MMFR1, LO, false, MRS_LOWER, 0, id_aa64mmfr1_lo), 1411 MRS_FIELD(ID_AA64MMFR1, HPDS, false, MRS_LOWER, 0, id_aa64mmfr1_hpds), 1412 MRS_FIELD(ID_AA64MMFR1, VH, false, MRS_LOWER, 0, id_aa64mmfr1_vh), 1413 MRS_FIELD(ID_AA64MMFR1, VMIDBits, false, MRS_LOWER, 0, 1414 id_aa64mmfr1_vmidbits), 1415 MRS_FIELD(ID_AA64MMFR1, HAFDBS, false, MRS_LOWER, 0, id_aa64mmfr1_hafdbs), 1416 MRS_FIELD_END, 1417 }; 1418 1419 1420 /* ID_AA64MMFR2_EL1 */ 1421 static const struct mrs_field_value id_aa64mmfr2_e0pd[] = { 1422 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, E0PD, NONE, IMPL), 1423 MRS_FIELD_VALUE_END, 1424 }; 1425 1426 static const struct mrs_field_value id_aa64mmfr2_evt[] = { 1427 MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_NONE, ""), 1428 MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_8_2, "EVT-8.2"), 1429 MRS_FIELD_VALUE(ID_AA64MMFR2_EVT_8_5, "EVT-8.5"), 1430 MRS_FIELD_VALUE_END, 1431 }; 1432 1433 static const struct mrs_field_value id_aa64mmfr2_bbm[] = { 1434 MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL0, ""), 1435 MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL1, "BBM level 1"), 1436 MRS_FIELD_VALUE(ID_AA64MMFR2_BBM_LEVEL2, "BBM level 2"), 1437 MRS_FIELD_VALUE_END, 1438 }; 1439 1440 static const struct mrs_field_value id_aa64mmfr2_ttl[] = { 1441 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, TTL, NONE, IMPL), 1442 MRS_FIELD_VALUE_END, 1443 }; 1444 1445 static const struct mrs_field_value id_aa64mmfr2_fwb[] = { 1446 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, FWB, NONE, IMPL), 1447 MRS_FIELD_VALUE_END, 1448 }; 1449 1450 static const struct mrs_field_value id_aa64mmfr2_ids[] = { 1451 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, IDS, NONE, IMPL), 1452 MRS_FIELD_VALUE_END, 1453 }; 1454 1455 static const struct mrs_field_value id_aa64mmfr2_at[] = { 1456 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, AT, NONE, IMPL), 1457 MRS_FIELD_VALUE_END, 1458 }; 1459 1460 static const struct mrs_field_hwcap id_aa64mmfr2_at_caps[] = { 1461 MRS_HWCAP(1, HWCAP_USCAT, ID_AA64MMFR2_AT_IMPL), 1462 MRS_HWCAP_END 1463 }; 1464 1465 static const struct mrs_field_value id_aa64mmfr2_st[] = { 1466 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, ST, NONE, IMPL), 1467 MRS_FIELD_VALUE_END, 1468 }; 1469 1470 static const struct mrs_field_value id_aa64mmfr2_nv[] = { 1471 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, NV, NONE, 8_3), 1472 MRS_FIELD_VALUE(ID_AA64MMFR2_NV_8_4, "NV v8.4"), 1473 MRS_FIELD_VALUE_END, 1474 }; 1475 1476 static const struct mrs_field_value id_aa64mmfr2_ccidx[] = { 1477 MRS_FIELD_VALUE(ID_AA64MMFR2_CCIDX_32, "32bit CCIDX"), 1478 MRS_FIELD_VALUE(ID_AA64MMFR2_CCIDX_64, "64bit CCIDX"), 1479 MRS_FIELD_VALUE_END, 1480 }; 1481 1482 static const struct mrs_field_value id_aa64mmfr2_varange[] = { 1483 MRS_FIELD_VALUE(ID_AA64MMFR2_VARange_48, "48bit VA"), 1484 MRS_FIELD_VALUE(ID_AA64MMFR2_VARange_52, "52bit VA"), 1485 MRS_FIELD_VALUE_END, 1486 }; 1487 1488 static const struct mrs_field_value id_aa64mmfr2_iesb[] = { 1489 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, IESB, NONE, IMPL), 1490 MRS_FIELD_VALUE_END, 1491 }; 1492 1493 static const struct mrs_field_value id_aa64mmfr2_lsm[] = { 1494 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, LSM, NONE, IMPL), 1495 MRS_FIELD_VALUE_END, 1496 }; 1497 1498 static const struct mrs_field_value id_aa64mmfr2_uao[] = { 1499 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, UAO, NONE, IMPL), 1500 MRS_FIELD_VALUE_END, 1501 }; 1502 1503 static const struct mrs_field_value id_aa64mmfr2_cnp[] = { 1504 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR2, CnP, NONE, IMPL), 1505 MRS_FIELD_VALUE_END, 1506 }; 1507 1508 static const struct mrs_field id_aa64mmfr2_fields[] = { 1509 MRS_FIELD(ID_AA64MMFR2, E0PD, false, MRS_LOWER, 0, id_aa64mmfr2_e0pd), 1510 MRS_FIELD(ID_AA64MMFR2, EVT, false, MRS_LOWER, 0, id_aa64mmfr2_evt), 1511 MRS_FIELD(ID_AA64MMFR2, BBM, false, MRS_LOWER, 0, id_aa64mmfr2_bbm), 1512 MRS_FIELD(ID_AA64MMFR2, TTL, false, MRS_LOWER, 0, id_aa64mmfr2_ttl), 1513 MRS_FIELD(ID_AA64MMFR2, FWB, false, MRS_LOWER, 0, id_aa64mmfr2_fwb), 1514 MRS_FIELD(ID_AA64MMFR2, IDS, false, MRS_LOWER, 0, id_aa64mmfr2_ids), 1515 MRS_FIELD_HWCAP(ID_AA64MMFR2, AT, false, MRS_LOWER, MRS_USERSPACE, 1516 id_aa64mmfr2_at, id_aa64mmfr2_at_caps), 1517 MRS_FIELD(ID_AA64MMFR2, ST, false, MRS_LOWER, 0, id_aa64mmfr2_st), 1518 MRS_FIELD(ID_AA64MMFR2, NV, false, MRS_LOWER, 0, id_aa64mmfr2_nv), 1519 MRS_FIELD(ID_AA64MMFR2, CCIDX, false, MRS_LOWER, 0, id_aa64mmfr2_ccidx), 1520 MRS_FIELD(ID_AA64MMFR2, VARange, false, MRS_LOWER, 0, 1521 id_aa64mmfr2_varange), 1522 MRS_FIELD(ID_AA64MMFR2, IESB, false, MRS_LOWER, 0, id_aa64mmfr2_iesb), 1523 MRS_FIELD(ID_AA64MMFR2, LSM, false, MRS_LOWER, 0, id_aa64mmfr2_lsm), 1524 MRS_FIELD(ID_AA64MMFR2, UAO, false, MRS_LOWER, 0, id_aa64mmfr2_uao), 1525 MRS_FIELD(ID_AA64MMFR2, CnP, false, MRS_LOWER, 0, id_aa64mmfr2_cnp), 1526 MRS_FIELD_END, 1527 }; 1528 1529 1530 /* ID_AA64MMFR2_EL1 */ 1531 static const struct mrs_field_value id_aa64mmfr3_spec_fpacc[] = { 1532 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, Spec_FPACC, NONE, IMPL), 1533 MRS_FIELD_VALUE_END, 1534 }; 1535 1536 static const struct mrs_field_value id_aa64mmfr3_aderr[] = { 1537 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, ADERR, NONE, SOME), 1538 MRS_FIELD_VALUE_END, 1539 }; 1540 1541 static const struct mrs_field_value id_aa64mmfr3_sderr[] = { 1542 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, SDERR, NONE, ALL), 1543 MRS_FIELD_VALUE_END, 1544 }; 1545 1546 static const struct mrs_field_value id_aa64mmfr3_anerr[] = { 1547 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, ANERR, NONE, SOME), 1548 MRS_FIELD_VALUE_END, 1549 }; 1550 1551 static const struct mrs_field_value id_aa64mmfr3_snerr[] = { 1552 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, SNERR, NONE, ALL), 1553 MRS_FIELD_VALUE_END, 1554 }; 1555 1556 static const struct mrs_field_value id_aa64mmfr3_mec[] = { 1557 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, MEC, NONE, IMPL), 1558 MRS_FIELD_VALUE_END, 1559 }; 1560 1561 static const struct mrs_field_value id_aa64mmfr3_aie[] = { 1562 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, AIE, NONE, IMPL), 1563 MRS_FIELD_VALUE_END, 1564 }; 1565 1566 static const struct mrs_field_value id_aa64mmfr3_s2poe[] = { 1567 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, S2POE, NONE, IMPL), 1568 MRS_FIELD_VALUE_END, 1569 }; 1570 1571 static const struct mrs_field_value id_aa64mmfr3_s1poe[] = { 1572 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, S1POE, NONE, IMPL), 1573 MRS_FIELD_VALUE_END, 1574 }; 1575 1576 static const struct mrs_field_value id_aa64mmfr3_s2pie[] = { 1577 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, S2PIE, NONE, IMPL), 1578 MRS_FIELD_VALUE_END, 1579 }; 1580 1581 static const struct mrs_field_value id_aa64mmfr3_s1pie[] = { 1582 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, S1PIE, NONE, IMPL), 1583 MRS_FIELD_VALUE_END, 1584 }; 1585 1586 static const struct mrs_field_value id_aa64mmfr3_sctlrx[] = { 1587 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, SCTLRX, NONE, IMPL), 1588 MRS_FIELD_VALUE_END, 1589 }; 1590 1591 static const struct mrs_field_value id_aa64mmfr3_tcrx[] = { 1592 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64MMFR3, TCRX, NONE, IMPL), 1593 MRS_FIELD_VALUE_END, 1594 }; 1595 1596 static const struct mrs_field id_aa64mmfr3_fields[] = { 1597 MRS_FIELD(ID_AA64MMFR3, Spec_FPACC, false, MRS_LOWER, 0, 1598 id_aa64mmfr3_spec_fpacc), 1599 MRS_FIELD(ID_AA64MMFR3, ADERR, false, MRS_LOWER, 0, id_aa64mmfr3_aderr), 1600 MRS_FIELD(ID_AA64MMFR3, SDERR, false, MRS_LOWER, 0, id_aa64mmfr3_sderr), 1601 MRS_FIELD(ID_AA64MMFR3, ANERR, false, MRS_LOWER, 0, id_aa64mmfr3_anerr), 1602 MRS_FIELD(ID_AA64MMFR3, SNERR, false, MRS_LOWER, 0, id_aa64mmfr3_snerr), 1603 MRS_FIELD(ID_AA64MMFR3, MEC, false, MRS_LOWER, 0, id_aa64mmfr3_mec), 1604 MRS_FIELD(ID_AA64MMFR3, AIE, false, MRS_LOWER, 0, id_aa64mmfr3_aie), 1605 MRS_FIELD(ID_AA64MMFR3, S2POE, false, MRS_LOWER, 0, id_aa64mmfr3_s2poe), 1606 MRS_FIELD(ID_AA64MMFR3, S1POE, false, MRS_LOWER, 0, id_aa64mmfr3_s1poe), 1607 MRS_FIELD(ID_AA64MMFR3, S2PIE, false, MRS_LOWER, 0, id_aa64mmfr3_s2pie), 1608 MRS_FIELD(ID_AA64MMFR3, S1PIE, false, MRS_LOWER, 0, id_aa64mmfr3_s1pie), 1609 MRS_FIELD(ID_AA64MMFR3, SCTLRX, false, MRS_LOWER, 0, 1610 id_aa64mmfr3_sctlrx), 1611 MRS_FIELD(ID_AA64MMFR3, TCRX, false, MRS_LOWER, 0, id_aa64mmfr3_tcrx), 1612 MRS_FIELD_END, 1613 }; 1614 1615 1616 /* ID_AA64MMFR4_EL1 */ 1617 static const struct mrs_field id_aa64mmfr4_fields[] = { 1618 MRS_FIELD_END, 1619 }; 1620 1621 1622 /* ID_AA64PFR0_EL1 */ 1623 static const struct mrs_field_value id_aa64pfr0_csv3[] = { 1624 MRS_FIELD_VALUE(ID_AA64PFR0_CSV3_NONE, ""), 1625 MRS_FIELD_VALUE(ID_AA64PFR0_CSV3_ISOLATED, "CSV3"), 1626 MRS_FIELD_VALUE_END, 1627 }; 1628 1629 static const struct mrs_field_value id_aa64pfr0_csv2[] = { 1630 MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_NONE, ""), 1631 MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_ISOLATED, "CSV2"), 1632 MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_SCXTNUM, "CSV2_2"), 1633 MRS_FIELD_VALUE(ID_AA64PFR0_CSV2_3, "CSV2_3"), 1634 MRS_FIELD_VALUE_END, 1635 }; 1636 1637 static const struct mrs_field_value id_aa64pfr0_rme[] = { 1638 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, RME, NONE, IMPL), 1639 MRS_FIELD_VALUE_END, 1640 }; 1641 1642 static const struct mrs_field_value id_aa64pfr0_dit[] = { 1643 MRS_FIELD_VALUE(ID_AA64PFR0_DIT_NONE, ""), 1644 MRS_FIELD_VALUE(ID_AA64PFR0_DIT_PSTATE, "PSTATE.DIT"), 1645 MRS_FIELD_VALUE_END, 1646 }; 1647 1648 static const struct mrs_field_hwcap id_aa64pfr0_dit_caps[] = { 1649 MRS_HWCAP(1, HWCAP_DIT, ID_AA64PFR0_DIT_PSTATE), 1650 MRS_HWCAP_END 1651 }; 1652 1653 static const struct mrs_field_value id_aa64pfr0_amu[] = { 1654 MRS_FIELD_VALUE(ID_AA64PFR0_AMU_NONE, ""), 1655 MRS_FIELD_VALUE(ID_AA64PFR0_AMU_V1, "AMUv1"), 1656 MRS_FIELD_VALUE(ID_AA64PFR0_AMU_V1_1, "AMUv1p1"), 1657 MRS_FIELD_VALUE_END, 1658 }; 1659 1660 static const struct mrs_field_value id_aa64pfr0_mpam[] = { 1661 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, MPAM, NONE, IMPL), 1662 MRS_FIELD_VALUE_END, 1663 }; 1664 1665 static const struct mrs_field_value id_aa64pfr0_sel2[] = { 1666 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, SEL2, NONE, IMPL), 1667 MRS_FIELD_VALUE_END, 1668 }; 1669 1670 static const struct mrs_field_value id_aa64pfr0_sve[] = { 1671 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, SVE, NONE, IMPL), 1672 MRS_FIELD_VALUE_END, 1673 }; 1674 1675 static const struct mrs_field_hwcap id_aa64pfr0_sve_caps[] = { 1676 MRS_HWCAP(1, HWCAP_SVE, ID_AA64PFR0_SVE_IMPL), 1677 MRS_HWCAP_END 1678 }; 1679 1680 static const struct mrs_field_value id_aa64pfr0_ras[] = { 1681 MRS_FIELD_VALUE(ID_AA64PFR0_RAS_NONE, ""), 1682 MRS_FIELD_VALUE(ID_AA64PFR0_RAS_IMPL, "RAS"), 1683 MRS_FIELD_VALUE(ID_AA64PFR0_RAS_8_4, "RAS v8.4"), 1684 MRS_FIELD_VALUE(ID_AA64PFR0_RAS_8_9, "RAS v8.9"), 1685 MRS_FIELD_VALUE_END, 1686 }; 1687 1688 static const struct mrs_field_value id_aa64pfr0_gic[] = { 1689 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, GIC, CPUIF_NONE, CPUIF_EN), 1690 MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_NONE, ""), 1691 MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_EN, "GIC"), 1692 MRS_FIELD_VALUE(ID_AA64PFR0_GIC_CPUIF_4_1, "GIC 4.1"), 1693 MRS_FIELD_VALUE_END, 1694 }; 1695 1696 static const struct mrs_field_value id_aa64pfr0_advsimd[] = { 1697 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, AdvSIMD, NONE, IMPL), 1698 MRS_FIELD_VALUE(ID_AA64PFR0_AdvSIMD_HP, "AdvSIMD+HP"), 1699 MRS_FIELD_VALUE_END, 1700 }; 1701 1702 static const struct mrs_field_hwcap id_aa64pfr0_advsimd_caps[] = { 1703 MRS_HWCAP(1, HWCAP_ASIMD, ID_AA64PFR0_AdvSIMD_IMPL), 1704 MRS_HWCAP(1, HWCAP_ASIMDHP, ID_AA64PFR0_AdvSIMD_HP), 1705 MRS_HWCAP_END 1706 }; 1707 1708 static const struct mrs_field_value id_aa64pfr0_fp[] = { 1709 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, FP, NONE, IMPL), 1710 MRS_FIELD_VALUE(ID_AA64PFR0_FP_HP, "FP+HP"), 1711 MRS_FIELD_VALUE_END, 1712 }; 1713 1714 static const struct mrs_field_hwcap id_aa64pfr0_fp_caps[] = { 1715 MRS_HWCAP(1, HWCAP_FP, ID_AA64PFR0_FP_IMPL), 1716 MRS_HWCAP(1, HWCAP_FPHP, ID_AA64PFR0_FP_HP), 1717 MRS_HWCAP_END 1718 }; 1719 1720 static const struct mrs_field_value id_aa64pfr0_el3[] = { 1721 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, EL3, NONE, 64), 1722 MRS_FIELD_VALUE(ID_AA64PFR0_EL3_64_32, "EL3 32"), 1723 MRS_FIELD_VALUE_END, 1724 }; 1725 1726 static const struct mrs_field_value id_aa64pfr0_el2[] = { 1727 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR0, EL2, NONE, 64), 1728 MRS_FIELD_VALUE(ID_AA64PFR0_EL2_64_32, "EL2 32"), 1729 MRS_FIELD_VALUE_END, 1730 }; 1731 1732 static const struct mrs_field_value id_aa64pfr0_el1[] = { 1733 MRS_FIELD_VALUE(ID_AA64PFR0_EL1_64, "EL1"), 1734 MRS_FIELD_VALUE(ID_AA64PFR0_EL1_64_32, "EL1 32"), 1735 MRS_FIELD_VALUE_END, 1736 }; 1737 1738 static const struct mrs_field_value id_aa64pfr0_el0[] = { 1739 MRS_FIELD_VALUE(ID_AA64PFR0_EL0_64, "EL0"), 1740 MRS_FIELD_VALUE(ID_AA64PFR0_EL0_64_32, "EL0 32"), 1741 MRS_FIELD_VALUE_END, 1742 }; 1743 1744 static const struct mrs_field id_aa64pfr0_fields[] = { 1745 MRS_FIELD(ID_AA64PFR0, CSV3, false, MRS_LOWER, 0, id_aa64pfr0_csv3), 1746 MRS_FIELD(ID_AA64PFR0, CSV2, false, MRS_LOWER, 0, id_aa64pfr0_csv2), 1747 MRS_FIELD(ID_AA64PFR0, RME, false, MRS_LOWER, 0, id_aa64pfr0_rme), 1748 MRS_FIELD_HWCAP(ID_AA64PFR0, DIT, false, MRS_LOWER, MRS_USERSPACE, 1749 id_aa64pfr0_dit, id_aa64pfr0_dit_caps), 1750 MRS_FIELD(ID_AA64PFR0, AMU, false, MRS_LOWER, 0, id_aa64pfr0_amu), 1751 MRS_FIELD(ID_AA64PFR0, MPAM, false, MRS_LOWER, 0, id_aa64pfr0_mpam), 1752 MRS_FIELD(ID_AA64PFR0, SEL2, false, MRS_LOWER, 0, id_aa64pfr0_sel2), 1753 MRS_FIELD_HWCAP(ID_AA64PFR0, SVE, false, MRS_LOWER, 1754 MRS_FREEBSD, id_aa64pfr0_sve, id_aa64pfr0_sve_caps), 1755 MRS_FIELD(ID_AA64PFR0, RAS, false, MRS_LOWER, 0, id_aa64pfr0_ras), 1756 MRS_FIELD(ID_AA64PFR0, GIC, false, MRS_LOWER, 0, id_aa64pfr0_gic), 1757 MRS_FIELD_HWCAP(ID_AA64PFR0, AdvSIMD, true, MRS_LOWER, MRS_USERSPACE, 1758 id_aa64pfr0_advsimd, id_aa64pfr0_advsimd_caps), 1759 MRS_FIELD_HWCAP(ID_AA64PFR0, FP, true, MRS_LOWER, MRS_USERSPACE, 1760 id_aa64pfr0_fp, id_aa64pfr0_fp_caps), 1761 MRS_FIELD(ID_AA64PFR0, EL3, false, MRS_LOWER, 0, id_aa64pfr0_el3), 1762 MRS_FIELD(ID_AA64PFR0, EL2, false, MRS_LOWER, 0, id_aa64pfr0_el2), 1763 MRS_FIELD(ID_AA64PFR0, EL1, false, MRS_LOWER, MRS_USERSPACE, 1764 id_aa64pfr0_el1), 1765 MRS_FIELD(ID_AA64PFR0, EL0, false, MRS_LOWER, MRS_USERSPACE, 1766 id_aa64pfr0_el0), 1767 MRS_FIELD_END, 1768 }; 1769 1770 1771 /* ID_AA64PFR1_EL1 */ 1772 static const struct mrs_field_value id_aa64pfr1_pfar[] = { 1773 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR1, PFAR, NONE, IMPL), 1774 MRS_FIELD_VALUE_END, 1775 }; 1776 1777 static const struct mrs_field_value id_aa64pfr1_df2[] = { 1778 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR1, DF2, NONE, IMPL), 1779 MRS_FIELD_VALUE_END, 1780 }; 1781 1782 static const struct mrs_field_value id_aa64pfr1_mtex[] = { 1783 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR1, MTEX, NONE, IMPL), 1784 MRS_FIELD_VALUE_END, 1785 }; 1786 1787 static const struct mrs_field_value id_aa64pfr1_the[] = { 1788 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR1, THE, NONE, IMPL), 1789 MRS_FIELD_VALUE_END, 1790 }; 1791 1792 static const struct mrs_field_value id_aa64pfr1_mtefrac[] = { 1793 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR1, MTE_frac, NONE, IMPL), 1794 MRS_FIELD_VALUE_END, 1795 }; 1796 1797 static const struct mrs_field_value id_aa64pfr1_nmi[] = { 1798 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR1, NMI, NONE, IMPL), 1799 MRS_FIELD_VALUE_END, 1800 }; 1801 1802 static const struct mrs_field_value id_aa64pfr1_csv2_frac[] = { 1803 MRS_FIELD_VALUE(ID_AA64PFR1_CSV2_frac_p0, ""), 1804 MRS_FIELD_VALUE(ID_AA64PFR1_CSV2_frac_p1, "CSV2 p1"), 1805 MRS_FIELD_VALUE(ID_AA64PFR1_CSV2_frac_p2, "CSV2 p2"), 1806 MRS_FIELD_VALUE_END, 1807 }; 1808 1809 static const struct mrs_field_value id_aa64pfr1_rndr_trap[] = { 1810 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64PFR1, RNDR_trap, NONE, IMPL), 1811 MRS_FIELD_VALUE_END, 1812 }; 1813 1814 static const struct mrs_field_value id_aa64pfr1_sme[] = { 1815 MRS_FIELD_VALUE(ID_AA64PFR1_SME_NONE, ""), 1816 MRS_FIELD_VALUE(ID_AA64PFR1_SME_SME, "SME"), 1817 MRS_FIELD_VALUE(ID_AA64PFR1_SME_SME2, "SME2"), 1818 MRS_FIELD_VALUE_END, 1819 }; 1820 1821 static const struct mrs_field_value id_aa64pfr1_mpam_frac[] = { 1822 MRS_FIELD_VALUE(ID_AA64PFR1_MPAM_frac_p0, ""), 1823 MRS_FIELD_VALUE(ID_AA64PFR1_MPAM_frac_p1, "MPAM p1"), 1824 MRS_FIELD_VALUE_END, 1825 }; 1826 1827 static const struct mrs_field_value id_aa64pfr1_ras_frac[] = { 1828 MRS_FIELD_VALUE(ID_AA64PFR1_RAS_frac_p0, ""), 1829 MRS_FIELD_VALUE(ID_AA64PFR1_RAS_frac_p1, "RAS p1"), 1830 MRS_FIELD_VALUE_END, 1831 }; 1832 1833 static const struct mrs_field_value id_aa64pfr1_mte[] = { 1834 MRS_FIELD_VALUE(ID_AA64PFR1_MTE_NONE, ""), 1835 MRS_FIELD_VALUE(ID_AA64PFR1_MTE_MTE, "MTE"), 1836 MRS_FIELD_VALUE(ID_AA64PFR1_MTE_MTE2, "MTE2"), 1837 MRS_FIELD_VALUE(ID_AA64PFR1_MTE_MTE3, "MTE3"), 1838 MRS_FIELD_VALUE_END, 1839 }; 1840 1841 static const struct mrs_field_value id_aa64pfr1_ssbs[] = { 1842 MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_NONE, ""), 1843 MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_PSTATE, "PSTATE.SSBS"), 1844 MRS_FIELD_VALUE(ID_AA64PFR1_SSBS_PSTATE_MSR, "PSTATE.SSBS MSR"), 1845 MRS_FIELD_VALUE_END, 1846 }; 1847 1848 static const struct mrs_field_hwcap id_aa64pfr1_ssbs_caps[] = { 1849 MRS_HWCAP(1, HWCAP_SSBS, ID_AA64PFR1_SSBS_PSTATE), 1850 MRS_HWCAP_END 1851 }; 1852 1853 static const struct mrs_field_value id_aa64pfr1_bt[] = { 1854 MRS_FIELD_VALUE(ID_AA64PFR1_BT_NONE, ""), 1855 MRS_FIELD_VALUE(ID_AA64PFR1_BT_IMPL, "BTI"), 1856 MRS_FIELD_VALUE_END, 1857 }; 1858 1859 static const struct mrs_field_hwcap id_aa64pfr1_bt_caps[] = { 1860 MRS_HWCAP(2, HWCAP2_BTI, ID_AA64PFR1_BT_IMPL), 1861 MRS_HWCAP_END 1862 }; 1863 1864 static const struct mrs_field id_aa64pfr1_fields[] = { 1865 MRS_FIELD(ID_AA64PFR1, PFAR, false, MRS_LOWER, 0, id_aa64pfr1_pfar), 1866 MRS_FIELD(ID_AA64PFR1, DF2, false, MRS_LOWER, 0, id_aa64pfr1_df2), 1867 MRS_FIELD(ID_AA64PFR1, MTEX, false, MRS_LOWER, 0, id_aa64pfr1_mtex), 1868 MRS_FIELD(ID_AA64PFR1, THE, false, MRS_LOWER, 0, id_aa64pfr1_the), 1869 MRS_FIELD(ID_AA64PFR1, MTE_frac, true, MRS_LOWER, 0, 1870 id_aa64pfr1_mtefrac), 1871 MRS_FIELD(ID_AA64PFR1, NMI, false, MRS_LOWER, 0, id_aa64pfr1_nmi), 1872 MRS_FIELD(ID_AA64PFR1, CSV2_frac, false, MRS_LOWER, 0, 1873 id_aa64pfr1_csv2_frac), 1874 MRS_FIELD(ID_AA64PFR1, RNDR_trap, false, MRS_LOWER, 0, 1875 id_aa64pfr1_rndr_trap), 1876 MRS_FIELD(ID_AA64PFR1, SME, false, MRS_LOWER, 0, id_aa64pfr1_sme), 1877 MRS_FIELD(ID_AA64PFR1, MPAM_frac, false, MRS_LOWER, 0, 1878 id_aa64pfr1_mpam_frac), 1879 MRS_FIELD(ID_AA64PFR1, RAS_frac, false, MRS_LOWER, 0, 1880 id_aa64pfr1_ras_frac), 1881 MRS_FIELD(ID_AA64PFR1, MTE, false, MRS_LOWER, 0, id_aa64pfr1_mte), 1882 MRS_FIELD_HWCAP(ID_AA64PFR1, SSBS, false, MRS_LOWER, MRS_USERSPACE, 1883 id_aa64pfr1_ssbs, id_aa64pfr1_ssbs_caps), 1884 MRS_FIELD_HWCAP(ID_AA64PFR1, BT, false, MRS_LOWER, 1885 MRS_FREEBSD, id_aa64pfr1_bt, id_aa64pfr1_bt_caps), 1886 MRS_FIELD_END, 1887 }; 1888 1889 1890 /* ID_AA64PFR2_EL1 */ 1891 static const struct mrs_field id_aa64pfr2_fields[] = { 1892 MRS_FIELD_END, 1893 }; 1894 1895 1896 /* ID_AA64ZFR0_EL1 */ 1897 static const struct mrs_field_value id_aa64zfr0_f64mm[] = { 1898 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, F64MM, NONE, IMPL), 1899 MRS_FIELD_VALUE_END, 1900 }; 1901 1902 static const struct mrs_field_hwcap id_aa64zfr0_f64mm_caps[] = { 1903 MRS_HWCAP(2, HWCAP2_SVEF64MM, ID_AA64ZFR0_F64MM_IMPL), 1904 MRS_HWCAP_END, 1905 }; 1906 1907 static const struct mrs_field_value id_aa64zfr0_f32mm[] = { 1908 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, F32MM, NONE, IMPL), 1909 MRS_FIELD_VALUE_END, 1910 }; 1911 1912 static const struct mrs_field_hwcap id_aa64zfr0_f32mm_caps[] = { 1913 MRS_HWCAP(2, HWCAP2_SVEF32MM, ID_AA64ZFR0_F32MM_IMPL), 1914 MRS_HWCAP_END, 1915 }; 1916 1917 static const struct mrs_field_value id_aa64zfr0_i8mm[] = { 1918 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, I8MM, NONE, IMPL), 1919 MRS_FIELD_VALUE_END, 1920 }; 1921 1922 static const struct mrs_field_hwcap id_aa64zfr0_i8mm_caps[] = { 1923 MRS_HWCAP(2, HWCAP2_SVEI8MM, ID_AA64ZFR0_I8MM_IMPL), 1924 MRS_HWCAP_END, 1925 }; 1926 1927 static const struct mrs_field_value id_aa64zfr0_sm4[] = { 1928 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, SM4, NONE, IMPL), 1929 MRS_FIELD_VALUE_END, 1930 }; 1931 1932 static const struct mrs_field_hwcap id_aa64zfr0_sm4_caps[] = { 1933 MRS_HWCAP(2, HWCAP2_SVESM4, ID_AA64ZFR0_SM4_IMPL), 1934 MRS_HWCAP_END, 1935 }; 1936 1937 static const struct mrs_field_value id_aa64zfr0_sha3[] = { 1938 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, SHA3, NONE, IMPL), 1939 MRS_FIELD_VALUE_END, 1940 }; 1941 1942 static const struct mrs_field_hwcap id_aa64zfr0_sha3_caps[] = { 1943 MRS_HWCAP(2, HWCAP2_SVESHA3, ID_AA64ZFR0_SHA3_IMPL), 1944 MRS_HWCAP_END, 1945 }; 1946 1947 static const struct mrs_field_value id_aa64zfr0_bf16[] = { 1948 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, BF16, NONE, BASE), 1949 MRS_FIELD_VALUE(ID_AA64ZFR0_BF16_EBF, "BF16+EBF"), 1950 MRS_FIELD_VALUE_END, 1951 }; 1952 1953 static const struct mrs_field_hwcap id_aa64zfr0_bf16_caps[] = { 1954 MRS_HWCAP(2, HWCAP2_SVEBF16, ID_AA64ZFR0_BF16_BASE), 1955 MRS_HWCAP(2, HWCAP2_SVE_EBF16, ID_AA64ZFR0_BF16_EBF), 1956 MRS_HWCAP_END, 1957 }; 1958 1959 static const struct mrs_field_value id_aa64zfr0_bitperm[] = { 1960 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, BitPerm, NONE, IMPL), 1961 MRS_FIELD_VALUE_END, 1962 }; 1963 1964 static const struct mrs_field_hwcap id_aa64zfr0_bitperm_caps[] = { 1965 MRS_HWCAP(2, HWCAP2_SVEBITPERM, ID_AA64ZFR0_BitPerm_IMPL), 1966 MRS_HWCAP_END, 1967 }; 1968 1969 static const struct mrs_field_value id_aa64zfr0_aes[] = { 1970 MRS_FIELD_VALUE_NONE_IMPL(ID_AA64ZFR0, AES, NONE, BASE), 1971 MRS_FIELD_VALUE(ID_AA64ZFR0_AES_PMULL, "AES+PMULL"), 1972 MRS_FIELD_VALUE_END, 1973 }; 1974 1975 static const struct mrs_field_hwcap id_aa64zfr0_aes_caps[] = { 1976 MRS_HWCAP(2, HWCAP2_SVEAES, ID_AA64ZFR0_AES_BASE), 1977 MRS_HWCAP(2, HWCAP2_SVEPMULL, ID_AA64ZFR0_AES_PMULL), 1978 MRS_HWCAP_END, 1979 }; 1980 1981 static const struct mrs_field_value id_aa64zfr0_svever[] = { 1982 MRS_FIELD_VALUE(ID_AA64ZFR0_SVEver_SVE1, "SVE1"), 1983 MRS_FIELD_VALUE(ID_AA64ZFR0_SVEver_SVE2, "SVE2"), 1984 MRS_FIELD_VALUE(ID_AA64ZFR0_SVEver_SVE2P1, "SVE2P1"), 1985 MRS_FIELD_VALUE_END, 1986 }; 1987 1988 static const struct mrs_field_hwcap id_aa64zfr0_svever_caps[] = { 1989 MRS_HWCAP(2, HWCAP2_SVE2, ID_AA64ZFR0_SVEver_SVE2), 1990 MRS_HWCAP(2, HWCAP2_SVE2P1, ID_AA64ZFR0_SVEver_SVE2P1), 1991 MRS_HWCAP_END, 1992 }; 1993 1994 static const struct mrs_field id_aa64zfr0_fields[] = { 1995 MRS_FIELD_HWCAP(ID_AA64ZFR0, F64MM, false, MRS_LOWER, MRS_USERSPACE, 1996 id_aa64zfr0_f64mm, id_aa64zfr0_f64mm_caps), 1997 MRS_FIELD_HWCAP(ID_AA64ZFR0, F32MM, false, MRS_LOWER, MRS_USERSPACE, 1998 id_aa64zfr0_f32mm, id_aa64zfr0_f32mm_caps), 1999 MRS_FIELD_HWCAP(ID_AA64ZFR0, I8MM, false, MRS_LOWER, MRS_USERSPACE, 2000 id_aa64zfr0_i8mm, id_aa64zfr0_i8mm_caps), 2001 MRS_FIELD_HWCAP(ID_AA64ZFR0, SM4, false, MRS_LOWER, MRS_USERSPACE, 2002 id_aa64zfr0_sm4, id_aa64zfr0_sm4_caps), 2003 MRS_FIELD_HWCAP(ID_AA64ZFR0, SHA3, false, MRS_LOWER, MRS_USERSPACE, 2004 id_aa64zfr0_sha3, id_aa64zfr0_sha3_caps), 2005 MRS_FIELD_HWCAP(ID_AA64ZFR0, BF16, false, MRS_LOWER, MRS_USERSPACE, 2006 id_aa64zfr0_bf16, id_aa64zfr0_bf16_caps), 2007 MRS_FIELD_HWCAP(ID_AA64ZFR0, BitPerm, false, MRS_LOWER, MRS_USERSPACE, 2008 id_aa64zfr0_bitperm, id_aa64zfr0_bitperm_caps), 2009 MRS_FIELD_HWCAP(ID_AA64ZFR0, AES, false, MRS_LOWER, MRS_USERSPACE, 2010 id_aa64zfr0_aes, id_aa64zfr0_aes_caps), 2011 MRS_FIELD_HWCAP(ID_AA64ZFR0, SVEver, false, MRS_LOWER, MRS_USERSPACE, 2012 id_aa64zfr0_svever, id_aa64zfr0_svever_caps), 2013 MRS_FIELD_END, 2014 }; 2015 2016 2017 #ifdef COMPAT_FREEBSD32 2018 /* ID_ISAR5_EL1 */ 2019 static const struct mrs_field_value id_isar5_vcma[] = { 2020 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, VCMA, NONE, IMPL), 2021 MRS_FIELD_VALUE_END, 2022 }; 2023 2024 static const struct mrs_field_value id_isar5_rdm[] = { 2025 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, RDM, NONE, IMPL), 2026 MRS_FIELD_VALUE_END, 2027 }; 2028 2029 static const struct mrs_field_value id_isar5_crc32[] = { 2030 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, CRC32, NONE, IMPL), 2031 MRS_FIELD_VALUE_END, 2032 }; 2033 2034 static const struct mrs_field_hwcap id_isar5_crc32_caps[] = { 2035 MRS_HWCAP(2, HWCAP32_2_CRC32, ID_ISAR5_CRC32_IMPL), 2036 MRS_HWCAP_END 2037 }; 2038 2039 static const struct mrs_field_value id_isar5_sha2[] = { 2040 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SHA2, NONE, IMPL), 2041 MRS_FIELD_VALUE_END, 2042 }; 2043 2044 static const struct mrs_field_hwcap id_isar5_sha2_caps[] = { 2045 MRS_HWCAP(2, HWCAP32_2_SHA2, ID_ISAR5_SHA2_IMPL), 2046 MRS_HWCAP_END 2047 }; 2048 2049 static const struct mrs_field_value id_isar5_sha1[] = { 2050 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SHA1, NONE, IMPL), 2051 MRS_FIELD_VALUE_END, 2052 }; 2053 2054 static const struct mrs_field_hwcap id_isar5_sha1_caps[] = { 2055 MRS_HWCAP(2, HWCAP32_2_SHA1, ID_ISAR5_SHA1_IMPL), 2056 MRS_HWCAP_END 2057 }; 2058 2059 static const struct mrs_field_value id_isar5_aes[] = { 2060 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, AES, NONE, BASE), 2061 MRS_FIELD_VALUE(ID_ISAR5_AES_VMULL, "AES+VMULL"), 2062 MRS_FIELD_VALUE_END, 2063 }; 2064 2065 static const struct mrs_field_hwcap id_isar5_aes_caps[] = { 2066 MRS_HWCAP(2, HWCAP32_2_AES, ID_ISAR5_AES_BASE), 2067 MRS_HWCAP(2, HWCAP32_2_PMULL, ID_ISAR5_AES_VMULL), 2068 MRS_HWCAP_END 2069 }; 2070 2071 static const struct mrs_field_value id_isar5_sevl[] = { 2072 MRS_FIELD_VALUE_NONE_IMPL(ID_ISAR5, SEVL, NOP, IMPL), 2073 MRS_FIELD_VALUE_END, 2074 }; 2075 2076 static const struct mrs_field id_isar5_fields[] = { 2077 MRS_FIELD(ID_ISAR5, VCMA, false, MRS_LOWER,MRS_USERSPACE, 2078 id_isar5_vcma), 2079 MRS_FIELD(ID_ISAR5, RDM, false, MRS_LOWER, MRS_USERSPACE, id_isar5_rdm), 2080 MRS_FIELD_HWCAP(ID_ISAR5, CRC32, false, MRS_LOWER, MRS_USERSPACE, 2081 id_isar5_crc32, id_isar5_crc32_caps), 2082 MRS_FIELD_HWCAP(ID_ISAR5, SHA2, false, MRS_LOWER, MRS_USERSPACE, 2083 id_isar5_sha2, id_isar5_sha2_caps), 2084 MRS_FIELD_HWCAP(ID_ISAR5, SHA1, false, MRS_LOWER, MRS_USERSPACE, 2085 id_isar5_sha1, id_isar5_sha1_caps), 2086 MRS_FIELD_HWCAP(ID_ISAR5, AES, false, MRS_LOWER, MRS_USERSPACE, 2087 id_isar5_aes, id_isar5_aes_caps), 2088 MRS_FIELD(ID_ISAR5, SEVL, false, MRS_LOWER, MRS_USERSPACE, 2089 id_isar5_sevl), 2090 MRS_FIELD_END, 2091 }; 2092 2093 /* MVFR0 */ 2094 static const struct mrs_field_value mvfr0_fpround[] = { 2095 MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPRound, NONE, IMPL), 2096 MRS_FIELD_VALUE_END, 2097 }; 2098 2099 static const struct mrs_field_value mvfr0_fpsqrt[] = { 2100 MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPSqrt, NONE, IMPL), 2101 MRS_FIELD_VALUE_END, 2102 }; 2103 2104 static const struct mrs_field_value mvfr0_fpdivide[] = { 2105 MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPDivide, NONE, IMPL), 2106 MRS_FIELD_VALUE_END, 2107 }; 2108 2109 static const struct mrs_field_value mvfr0_fptrap[] = { 2110 MRS_FIELD_VALUE_NONE_IMPL(MVFR0, FPTrap, NONE, IMPL), 2111 MRS_FIELD_VALUE_END, 2112 }; 2113 2114 static const struct mrs_field_value mvfr0_fpdp[] = { 2115 MRS_FIELD_VALUE(MVFR0_FPDP_NONE, ""), 2116 MRS_FIELD_VALUE(MVFR0_FPDP_VFP_v2, "DP VFPv2"), 2117 MRS_FIELD_VALUE(MVFR0_FPDP_VFP_v3_v4, "DP VFPv3+v4"), 2118 MRS_FIELD_VALUE_END, 2119 }; 2120 2121 static const struct mrs_field_hwcap mvfr0_fpdp_caps[] = { 2122 MRS_HWCAP(1, HWCAP32_VFP, MVFR0_FPDP_VFP_v2), 2123 MRS_HWCAP(1, HWCAP32_VFPv3, MVFR0_FPDP_VFP_v3_v4), 2124 MRS_HWCAP_END 2125 }; 2126 2127 static const struct mrs_field_value mvfr0_fpsp[] = { 2128 MRS_FIELD_VALUE(MVFR0_FPSP_NONE, ""), 2129 MRS_FIELD_VALUE(MVFR0_FPSP_VFP_v2, "SP VFPv2"), 2130 MRS_FIELD_VALUE(MVFR0_FPSP_VFP_v3_v4, "SP VFPv3+v4"), 2131 MRS_FIELD_VALUE_END, 2132 }; 2133 2134 static const struct mrs_field_value mvfr0_simdreg[] = { 2135 MRS_FIELD_VALUE(MVFR0_SIMDReg_NONE, ""), 2136 MRS_FIELD_VALUE(MVFR0_SIMDReg_FP, "FP 16x64"), 2137 MRS_FIELD_VALUE(MVFR0_SIMDReg_AdvSIMD, "AdvSIMD"), 2138 MRS_FIELD_VALUE_END, 2139 }; 2140 2141 static const struct mrs_field mvfr0_fields[] = { 2142 MRS_FIELD(MVFR0, FPRound, false, MRS_LOWER, MRS_USERSPACE, 2143 mvfr0_fpround), 2144 MRS_FIELD(MVFR0, FPSqrt, false, MRS_LOWER, MRS_USERSPACE, 2145 mvfr0_fpsqrt), 2146 MRS_FIELD(MVFR0, FPDivide, false, MRS_LOWER, MRS_USERSPACE, 2147 mvfr0_fpdivide), 2148 MRS_FIELD(MVFR0, FPTrap, false, MRS_LOWER, MRS_USERSPACE, 2149 mvfr0_fptrap), 2150 MRS_FIELD_HWCAP(MVFR0, FPDP, false, MRS_LOWER, MRS_USERSPACE, 2151 mvfr0_fpdp, mvfr0_fpdp_caps), 2152 MRS_FIELD(MVFR0, FPSP, false, MRS_LOWER, MRS_USERSPACE, mvfr0_fpsp), 2153 MRS_FIELD(MVFR0, SIMDReg, false, MRS_LOWER, MRS_USERSPACE, 2154 mvfr0_simdreg), 2155 MRS_FIELD_END, 2156 }; 2157 2158 /* MVFR1 */ 2159 static const struct mrs_field_value mvfr1_simdfmac[] = { 2160 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDFMAC, NONE, IMPL), 2161 MRS_FIELD_VALUE_END, 2162 }; 2163 2164 static const struct mrs_field_hwcap mvfr1_simdfmac_caps[] = { 2165 MRS_HWCAP(1, HWCAP32_VFPv4, MVFR1_SIMDFMAC_IMPL), 2166 MRS_HWCAP_END 2167 }; 2168 2169 static const struct mrs_field_value mvfr1_fphp[] = { 2170 MRS_FIELD_VALUE(MVFR1_FPHP_NONE, ""), 2171 MRS_FIELD_VALUE(MVFR1_FPHP_CONV_SP, "FPHP SP Conv"), 2172 MRS_FIELD_VALUE(MVFR1_FPHP_CONV_DP, "FPHP DP Conv"), 2173 MRS_FIELD_VALUE(MVFR1_FPHP_ARITH, "FPHP Arith"), 2174 MRS_FIELD_VALUE_END, 2175 }; 2176 2177 static const struct mrs_field_value mvfr1_simdhp[] = { 2178 MRS_FIELD_VALUE(MVFR1_SIMDHP_NONE, ""), 2179 MRS_FIELD_VALUE(MVFR1_SIMDHP_CONV_SP, "SIMDHP SP Conv"), 2180 MRS_FIELD_VALUE(MVFR1_SIMDHP_ARITH, "SIMDHP Arith"), 2181 MRS_FIELD_VALUE_END, 2182 }; 2183 2184 static const struct mrs_field_value mvfr1_simdsp[] = { 2185 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDSP, NONE, IMPL), 2186 MRS_FIELD_VALUE_END, 2187 }; 2188 2189 static const struct mrs_field_value mvfr1_simdint[] = { 2190 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDInt, NONE, IMPL), 2191 MRS_FIELD_VALUE_END, 2192 }; 2193 2194 static const struct mrs_field_value mvfr1_simdls[] = { 2195 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, SIMDLS, NONE, IMPL), 2196 MRS_FIELD_VALUE_END, 2197 }; 2198 2199 static const struct mrs_field_hwcap mvfr1_simdls_caps[] = { 2200 MRS_HWCAP(1, HWCAP32_VFPv4, MVFR1_SIMDFMAC_IMPL), 2201 MRS_HWCAP_END 2202 }; 2203 2204 static const struct mrs_field_value mvfr1_fpdnan[] = { 2205 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, FPDNaN, NONE, IMPL), 2206 MRS_FIELD_VALUE_END, 2207 }; 2208 2209 static const struct mrs_field_value mvfr1_fpftz[] = { 2210 MRS_FIELD_VALUE_NONE_IMPL(MVFR1, FPFtZ, NONE, IMPL), 2211 MRS_FIELD_VALUE_END, 2212 }; 2213 2214 static const struct mrs_field mvfr1_fields[] = { 2215 MRS_FIELD_HWCAP(MVFR1, SIMDFMAC, false, MRS_LOWER, MRS_USERSPACE, 2216 mvfr1_simdfmac, mvfr1_simdfmac_caps), 2217 MRS_FIELD(MVFR1, FPHP, false, MRS_LOWER, MRS_USERSPACE, mvfr1_fphp), 2218 MRS_FIELD(MVFR1, SIMDHP, false, MRS_LOWER, MRS_USERSPACE, mvfr1_simdhp), 2219 MRS_FIELD(MVFR1, SIMDSP, false, MRS_LOWER, MRS_USERSPACE, mvfr1_simdsp), 2220 MRS_FIELD(MVFR1, SIMDInt, false, MRS_LOWER, MRS_USERSPACE, 2221 mvfr1_simdint), 2222 MRS_FIELD_HWCAP(MVFR1, SIMDLS, false, MRS_LOWER, MRS_USERSPACE, 2223 mvfr1_simdls, mvfr1_simdls_caps), 2224 MRS_FIELD(MVFR1, FPDNaN, false, MRS_LOWER, MRS_USERSPACE, 2225 mvfr1_fpdnan), 2226 MRS_FIELD(MVFR1, FPFtZ, false, MRS_LOWER, MRS_USERSPACE, 2227 mvfr1_fpftz), 2228 MRS_FIELD_END, 2229 }; 2230 #endif /* COMPAT_FREEBSD32 */ 2231 2232 struct mrs_user_reg { 2233 u_int iss; 2234 bool is64bit; 2235 size_t offset; 2236 const struct mrs_field *fields; 2237 }; 2238 2239 #define USER_REG(name, field_name, _is64bit) \ 2240 { \ 2241 .iss = name##_ISS, \ 2242 .offset = __offsetof(struct cpu_desc, field_name), \ 2243 .fields = field_name##_fields, \ 2244 .is64bit = _is64bit, \ 2245 } 2246 static const struct mrs_user_reg user_regs[] = { 2247 USER_REG(ID_AA64AFR0_EL1, id_aa64afr0, true), 2248 USER_REG(ID_AA64AFR1_EL1, id_aa64afr1, true), 2249 2250 USER_REG(ID_AA64DFR0_EL1, id_aa64dfr0, true), 2251 USER_REG(ID_AA64DFR1_EL1, id_aa64dfr1, true), 2252 2253 USER_REG(ID_AA64ISAR0_EL1, id_aa64isar0, true), 2254 USER_REG(ID_AA64ISAR1_EL1, id_aa64isar1, true), 2255 USER_REG(ID_AA64ISAR2_EL1, id_aa64isar2, true), 2256 2257 USER_REG(ID_AA64MMFR0_EL1, id_aa64mmfr0, true), 2258 USER_REG(ID_AA64MMFR1_EL1, id_aa64mmfr1, true), 2259 USER_REG(ID_AA64MMFR2_EL1, id_aa64mmfr2, true), 2260 USER_REG(ID_AA64MMFR3_EL1, id_aa64mmfr3, true), 2261 USER_REG(ID_AA64MMFR4_EL1, id_aa64mmfr4, true), 2262 2263 USER_REG(ID_AA64PFR0_EL1, id_aa64pfr0, true), 2264 USER_REG(ID_AA64PFR1_EL1, id_aa64pfr1, true), 2265 USER_REG(ID_AA64PFR2_EL1, id_aa64pfr2, true), 2266 2267 USER_REG(ID_AA64ZFR0_EL1, id_aa64zfr0, true), 2268 2269 USER_REG(CTR_EL0, ctr, true), 2270 2271 #ifdef COMPAT_FREEBSD32 2272 USER_REG(ID_ISAR5_EL1, id_isar5, false), 2273 2274 USER_REG(MVFR0_EL1, mvfr0, false), 2275 USER_REG(MVFR1_EL1, mvfr1, false), 2276 #endif /* COMPAT_FREEBSD32 */ 2277 }; 2278 2279 #define CPU_DESC_FIELD(desc, idx) \ 2280 *(uint64_t *)((char *)&(desc) + user_regs[(idx)].offset) 2281 2282 static bool 2283 user_ctr_has_neoverse_n1_1542419(uint32_t midr, uint64_t ctr) 2284 { 2285 /* 2286 * Neoverse-N1 erratum 1542419 2287 * Present in r3p0 - r4p0 2288 * Fixed in r4p1 2289 */ 2290 return (midr_check_var_part_range(midr, CPU_IMPL_ARM, 2291 CPU_PART_NEOVERSE_N1, 3, 0, 4, 0) && CTR_DIC_VAL(ctr) != 0); 2292 } 2293 2294 static cpu_feat_en 2295 user_ctr_check(const struct cpu_feat *feat __unused, u_int midr) 2296 { 2297 if (emulate_ctr) 2298 return (FEAT_DEFAULT_ENABLE); 2299 2300 if (user_ctr_has_neoverse_n1_1542419(midr, READ_SPECIALREG(ctr_el0))) 2301 return (FEAT_DEFAULT_ENABLE); 2302 2303 return (FEAT_ALWAYS_DISABLE); 2304 } 2305 2306 static bool 2307 user_ctr_has_errata(const struct cpu_feat *feat __unused, u_int midr, 2308 u_int **errata_list, u_int *errata_count) 2309 { 2310 if (user_ctr_has_neoverse_n1_1542419(midr, READ_SPECIALREG(ctr_el0))) { 2311 static u_int errata_id = 1542419; 2312 2313 *errata_list = &errata_id; 2314 *errata_count = 1; 2315 return (true); 2316 } 2317 2318 return (false); 2319 } 2320 2321 static bool 2322 user_ctr_enable(const struct cpu_feat *feat __unused, 2323 cpu_feat_errata errata_status, u_int *errata_list, u_int errata_count) 2324 { 2325 MPASS(emulate_ctr || errata_status != ERRATA_NONE); 2326 2327 /* 2328 * The Errata Management Firmware Interface may incorrectly mark 2329 * this as firmware mitigated. We should ignore that as there is 2330 * a kernel component to the mitigation. 2331 */ 2332 if (errata_status != ERRATA_NONE && PCPU_GET(cpuid) == 0 && 2333 cpu_feat_has_erratum(errata_list, errata_count, 1542419)) { 2334 /* Clear fields we will change */ 2335 user_cpu_desc.ctr &= ~(CTR_DIC_MASK | CTR_ILINE_WIDTH); 2336 2337 /* 2338 * Set DIC to none so userspace will execute an 'ic ivau' 2339 * instruction that can be trapped by EL3. 2340 */ 2341 user_cpu_desc.ctr |= CTR_DIC_NONE; 2342 /* 2343 * Set the i-cache line size to be page size to reduce the 2344 * number of times userspace needs to execute the 'ic ivau' 2345 * instruction. The ctr_el0.IminLine is log2 the number of 2346 * 4-byte words the instruction covers. As PAGE_SHIFT is log2 2347 * of the number of bytes in a page we need to subtract 2. 2348 */ 2349 user_cpu_desc.ctr |= (PAGE_SHIFT - 2) << CTR_ILINE_SHIFT; 2350 2351 l_user_cpu_desc.ctr = user_cpu_desc.ctr; 2352 } 2353 2354 WRITE_SPECIALREG(sctlr_el1, 2355 READ_SPECIALREG(sctlr_el1) & ~SCTLR_UCT); 2356 isb(); 2357 2358 return (true); 2359 } 2360 2361 CPU_FEAT(trap_ctr, "Trap CTR_EL0", 2362 user_ctr_check, user_ctr_has_errata, user_ctr_enable, NULL, 2363 CPU_FEAT_AFTER_DEV | CPU_FEAT_PER_CPU); 2364 2365 static bool 2366 user_ctr_handler(uint64_t esr, struct trapframe *frame) 2367 { 2368 uint64_t value; 2369 int reg; 2370 2371 if (ESR_ELx_EXCEPTION(esr) != EXCP_MSR) 2372 return (false); 2373 2374 /* Only support reading from ctr_el0 */ 2375 if ((esr & ISS_MSR_DIR) == 0) 2376 return (false); 2377 2378 /* Check if this is the ctr_el0 register */ 2379 if ((esr & ISS_MSR_REG_MASK) != CTR_EL0_ISS) 2380 return (false); 2381 2382 if (SV_CURPROC_ABI() == SV_ABI_FREEBSD) 2383 value = user_cpu_desc.ctr; 2384 else 2385 value = l_user_cpu_desc.ctr; 2386 /* 2387 * We will handle this instruction, move to the next so we 2388 * don't trap here again. 2389 */ 2390 frame->tf_elr += INSN_SIZE; 2391 2392 reg = ISS_MSR_Rt(esr); 2393 /* If reg is 31 then write to xzr, i.e. do nothing */ 2394 if (reg == 31) 2395 return (true); 2396 2397 if (reg < nitems(frame->tf_x)) 2398 frame->tf_x[reg] = value; 2399 else if (reg == 30) 2400 frame->tf_lr = value; 2401 2402 return (true); 2403 } 2404 2405 static bool 2406 user_idreg_handler(uint64_t esr, struct trapframe *frame) 2407 { 2408 uint64_t value; 2409 int reg; 2410 2411 if (ESR_ELx_EXCEPTION(esr) != EXCP_MSR) 2412 return (false); 2413 2414 /* Only support reading from ID registers */ 2415 if ((esr & ISS_MSR_DIR) == 0) 2416 return (false); 2417 2418 /* 2419 * This only handles the ID register space and a few registers that 2420 * are safe to pass through to userspace. 2421 * 2422 * These registers are all in the space op0 == 3, op1 == 0, 2423 * CRn == 0. We support the following CRm: 2424 * - CRm == 0: midr_el1, mpidr_el1, and revidr_el1. 2425 * - CRm in {4-7}: sanitized ID registers. 2426 * 2427 * Registers in the ID register space (CRm in {4-7}) are all 2428 * read-only and have either defined fields, or are read as 2429 * zero (RAZ). For these we return 0 for any unknown register. 2430 */ 2431 if (ISS_MSR_OP0(esr) != 3 || ISS_MSR_OP1(esr) != 0 || 2432 ISS_MSR_CRn(esr) != 0) 2433 return (false); 2434 2435 value = 0; 2436 if (ISS_MSR_CRm(esr) >= 4 && ISS_MSR_CRm(esr) <= 7) { 2437 for (int i = 0; i < nitems(user_regs); i++) { 2438 if (user_regs[i].iss == (esr & ISS_MSR_REG_MASK)) { 2439 if (SV_CURPROC_ABI() == SV_ABI_FREEBSD) 2440 value = CPU_DESC_FIELD(user_cpu_desc, i); 2441 else 2442 value = CPU_DESC_FIELD(l_user_cpu_desc, i); 2443 break; 2444 } 2445 } 2446 } else if (ISS_MSR_CRm(esr) == 0) { 2447 switch (ISS_MSR_OP2(esr)) { 2448 case 0: 2449 value = READ_SPECIALREG(midr_el1); 2450 break; 2451 case 5: 2452 value = READ_SPECIALREG(mpidr_el1); 2453 break; 2454 case 6: 2455 value = READ_SPECIALREG(revidr_el1); 2456 break; 2457 default: 2458 return (false); 2459 } 2460 } else { 2461 return (false); 2462 } 2463 2464 /* 2465 * We will handle this instruction, move to the next so we 2466 * don't trap here again. 2467 */ 2468 frame->tf_elr += INSN_SIZE; 2469 2470 reg = ISS_MSR_Rt(esr); 2471 /* If reg is 31 then write to xzr, i.e. do nothing */ 2472 if (reg == 31) 2473 return (1); 2474 2475 if (reg < nitems(frame->tf_x)) 2476 frame->tf_x[reg] = value; 2477 else if (reg == 30) 2478 frame->tf_lr = value; 2479 2480 return (true); 2481 } 2482 2483 /* 2484 * Compares two field values that may be signed or unsigned. 2485 * Returns: 2486 * < 0 when a is less than b 2487 * = 0 when a equals b 2488 * > 0 when a is greater than b 2489 */ 2490 static int 2491 mrs_field_cmp(uint64_t a, uint64_t b, u_int shift, int width, bool sign) 2492 { 2493 uint64_t mask; 2494 2495 KASSERT(width > 0 && width < 64, ("%s: Invalid width %d", __func__, 2496 width)); 2497 2498 mask = (1ul << width) - 1; 2499 /* Move the field to the lower bits */ 2500 a = (a >> shift) & mask; 2501 b = (b >> shift) & mask; 2502 2503 if (sign) { 2504 /* 2505 * The field is signed. Toggle the upper bit so the comparison 2506 * works on unsigned values as this makes positive numbers, 2507 * i.e. those with a 0 bit, larger than negative numbers, 2508 * i.e. those with a 1 bit, in an unsigned comparison. 2509 */ 2510 a ^= 1ul << (width - 1); 2511 b ^= 1ul << (width - 1); 2512 } 2513 2514 return (a - b); 2515 } 2516 2517 void 2518 get_kernel_reg_iss(u_int iss, uint64_t *val) 2519 { 2520 int i; 2521 2522 for (i = 0; i < nitems(user_regs); i++) { 2523 if (user_regs[i].iss == iss) { 2524 *val = CPU_DESC_FIELD(kern_cpu_desc, i); 2525 return; 2526 } 2527 } 2528 2529 panic("%s: Invalid register %x", __func__, iss); 2530 } 2531 2532 /* 2533 * Fetch the specified register's value, ensuring that individual field values 2534 * do not exceed those in the mask. 2535 */ 2536 void 2537 get_kernel_reg_iss_masked(u_int iss, uint64_t *valp, uint64_t mask) 2538 { 2539 const struct mrs_field *fields; 2540 uint64_t val; 2541 2542 for (int i = 0; i < nitems(user_regs); i++) { 2543 if (user_regs[i].iss == iss) { 2544 val = CPU_DESC_FIELD(kern_cpu_desc, i); 2545 fields = user_regs[i].fields; 2546 for (int j = 0; fields[j].type != 0; j++) { 2547 mask = update_special_reg_field(mask, 2548 fields[j].type, val, fields[j].width, 2549 fields[j].shift, fields[j].sign); 2550 } 2551 *valp = mask; 2552 return; 2553 } 2554 } 2555 2556 panic("%s: Invalid register %x", __func__, iss); 2557 } 2558 2559 bool 2560 get_user_reg_iss(u_int iss, uint64_t *val, bool fbsd) 2561 { 2562 int i; 2563 2564 for (i = 0; i < nitems(user_regs); i++) { 2565 if (user_regs[i].iss == iss) { 2566 if (fbsd) 2567 *val = CPU_DESC_FIELD(user_cpu_desc, i); 2568 else 2569 *val = CPU_DESC_FIELD(l_user_cpu_desc, i); 2570 return (true); 2571 } 2572 } 2573 2574 return (false); 2575 } 2576 2577 static uint64_t 2578 update_special_reg_field(uint64_t user_reg, u_int type, uint64_t value, 2579 u_int width, u_int shift, bool sign) 2580 { 2581 uint64_t cur, mask, new_val; 2582 2583 mask = ((1ul << width) - 1) << shift; 2584 cur = user_reg & mask; 2585 new_val = value & mask; 2586 2587 switch (type & MRS_TYPE_MASK) { 2588 case MRS_EXACT_IF_DIFFERENT: 2589 if (mrs_field_cmp(new_val, cur, shift, width, sign) == 0) 2590 break; 2591 /* FALLTHROUGH */ 2592 case MRS_EXACT: 2593 cur = (uint64_t)MRS_SAFE_VAL(type) << shift; 2594 break; 2595 case MRS_LOWER: 2596 if (mrs_field_cmp(new_val, cur, shift, width, sign) < 0) 2597 cur = new_val; 2598 break; 2599 case MRS_HIGHER_OR_ZERO: 2600 if (cur == 0 || new_val == 0) { 2601 cur = 0; 2602 break; 2603 } 2604 /* FALLTHROUGH */ 2605 case MRS_HIGHER: 2606 if (mrs_field_cmp(new_val, cur, shift, width, sign) > 0) 2607 cur = new_val; 2608 break; 2609 default: 2610 panic("Invalid field type: %d", type); 2611 } 2612 2613 user_reg &= ~mask; 2614 user_reg |= cur; 2615 2616 return (user_reg); 2617 } 2618 2619 static void 2620 clear_set_special_reg_idx(int idx, uint64_t clear, uint64_t set) 2621 { 2622 const struct mrs_field *fields; 2623 uint64_t k_old, k_new; 2624 uint64_t f_old, f_new; 2625 uint64_t l_old, l_new; 2626 2627 MPASS(idx < nitems(user_regs)); 2628 2629 k_old = CPU_DESC_FIELD(kern_cpu_desc, idx); 2630 k_new = (k_old & ~clear) | set; 2631 2632 f_old = CPU_DESC_FIELD(user_cpu_desc, idx); 2633 f_new = (f_old & ~clear) | set; 2634 2635 l_old = CPU_DESC_FIELD(l_user_cpu_desc, idx); 2636 l_new = (l_old & ~clear) | set; 2637 2638 fields = user_regs[idx].fields; 2639 for (int j = 0; fields[j].type != 0; j++) { 2640 u_int type; 2641 2642 /* Update the FreeBSD userspace ID register view */ 2643 type = ((fields[j].type & MRS_FREEBSD) != 0) ? 2644 fields[j].type : 2645 (MRS_EXACT | (fields[j].type & MRS_SAFE_MASK)); 2646 f_new = update_special_reg_field(f_new, 2647 type, f_old, fields[j].width, fields[j].shift, 2648 fields[j].sign); 2649 2650 /* Update the Linux userspace ID register view */ 2651 type = ((fields[j].type & MRS_LINUX) != 0) ? 2652 fields[j].type : 2653 (MRS_EXACT | (fields[j].type & MRS_SAFE_MASK)); 2654 l_new = update_special_reg_field(l_new, 2655 type, l_old, fields[j].width, fields[j].shift, 2656 fields[j].sign); 2657 2658 /* Update the kernel ID register view */ 2659 k_new = update_special_reg_field(k_new, 2660 fields[j].type, k_old, fields[j].width, 2661 fields[j].shift, fields[j].sign); 2662 } 2663 2664 CPU_DESC_FIELD(kern_cpu_desc, idx) = k_new; 2665 CPU_DESC_FIELD(user_cpu_desc, idx) = f_new; 2666 CPU_DESC_FIELD(l_user_cpu_desc, idx) = l_new; 2667 } 2668 2669 void 2670 update_special_regs(u_int cpu) 2671 { 2672 struct cpu_desc *desc; 2673 uint64_t value; 2674 int i; 2675 2676 if (cpu == 0) { 2677 /* Create a user visible cpu description with safe values */ 2678 memset_early(&user_cpu_desc, 0, sizeof(user_cpu_desc)); 2679 /* Safe values for these registers */ 2680 user_cpu_desc.id_aa64pfr0 = ID_AA64PFR0_AdvSIMD_NONE | 2681 ID_AA64PFR0_FP_NONE | ID_AA64PFR0_EL1_64 | 2682 ID_AA64PFR0_EL0_64; 2683 user_cpu_desc.id_aa64dfr0 = ID_AA64DFR0_DebugVer_8; 2684 /* Create the Linux user visible cpu description */ 2685 memcpy_early(&l_user_cpu_desc, &user_cpu_desc, 2686 sizeof(user_cpu_desc)); 2687 } 2688 2689 desc = get_cpu_desc(cpu); 2690 for (i = 0; i < nitems(user_regs); i++) { 2691 value = CPU_DESC_FIELD(*desc, i); 2692 if (cpu == 0) { 2693 CPU_DESC_FIELD(kern_cpu_desc, i) = value; 2694 CPU_DESC_FIELD(user_cpu_desc, i) = value; 2695 CPU_DESC_FIELD(l_user_cpu_desc, i) = value; 2696 } 2697 2698 clear_set_special_reg_idx(i, UINT64_MAX, value); 2699 } 2700 } 2701 2702 /* 2703 * Updates a special register in all views. This creates a copy of the 2704 * register then clears it and sets new bits. It will then compare this 2705 * with the old version as if it was the ID register for a new CPU. 2706 * 2707 * It is intended to let code that disables features, e.g. due to errata, 2708 * to clear the user visible field. 2709 * 2710 * This needs to be called before the HWCAPs are set. If called from a CPU 2711 * feature handler this safe to call from CPU_FEAT_EARLY_BOOT. It also needs 2712 * to be before link_elf_late_ireloc is called. As this is called after the 2713 * HWCAPs are set the check for these is enough. 2714 */ 2715 void 2716 update_special_reg_iss(u_int iss, uint64_t clear, uint64_t set) 2717 { 2718 MPASS(hwcaps_set == false); 2719 /* There is no locking here, so we only support changing this on CPU0 */ 2720 /* TODO: Add said locking */ 2721 MPASS(PCPU_GET(cpuid) == 0); 2722 2723 for (int i = 0; i < nitems(user_regs); i++) { 2724 if (user_regs[i].iss != iss) 2725 continue; 2726 2727 clear_set_special_reg_idx(i, clear, set); 2728 return; 2729 } 2730 } 2731 2732 void 2733 cpu_desc_init(void) 2734 { 2735 if (mp_ncpus == 1) 2736 return; 2737 2738 /* 2739 * Allocate memory for the non-boot CPUs to store their registers. 2740 * As this is indexed by CPU ID we need to allocate space for CPUs 2741 * 1 to mp_maxid. Because of this mp_maxid is already the correct 2742 * number of elements. 2743 */ 2744 cpu_desc = mallocarray(mp_maxid, sizeof(*cpu_desc), M_IDENTCPU, 2745 M_ZERO | M_WAITOK); 2746 } 2747 2748 /* HWCAP */ 2749 bool __read_frequently lse_supported = false; 2750 2751 bool __read_frequently icache_aliasing = false; 2752 bool __read_frequently icache_vmid = false; 2753 2754 int64_t dcache_line_size; /* The minimum D cache line size */ 2755 int64_t icache_line_size; /* The minimum I cache line size */ 2756 int64_t idcache_line_size; /* The minimum cache line size */ 2757 2758 /* 2759 * Find the values to export to userspace as AT_HWCAP and AT_HWCAP2. 2760 */ 2761 static void 2762 parse_cpu_features(bool is64bit, struct cpu_desc *cpu_desc, u_long *hwcap, 2763 u_long *hwcap2) 2764 { 2765 const struct mrs_field_hwcap *hwcaps; 2766 const struct mrs_field *fields; 2767 uint64_t min, reg; 2768 u_long *cur_hwcap; 2769 int i, j, k; 2770 2771 for (i = 0; i < nitems(user_regs); i++) { 2772 if (user_regs[i].is64bit != is64bit) 2773 continue; 2774 2775 reg = CPU_DESC_FIELD(*cpu_desc, i); 2776 fields = user_regs[i].fields; 2777 for (j = 0; fields[j].type != 0; j++) { 2778 hwcaps = fields[j].hwcaps; 2779 if (hwcaps == NULL) 2780 continue; 2781 2782 for (k = 0; hwcaps[k].hwcap_id != 0; k++) { 2783 KASSERT(hwcaps[k].hwcap_id == 1 || 2784 hwcaps[k].hwcap_id == 2, 2785 ("%s: Invalid HWCAP ID %d", __func__, 2786 hwcaps[k].hwcap_id)); 2787 2788 cur_hwcap = hwcaps[k].hwcap_id == 1 ? 2789 hwcap : hwcap2; 2790 min = hwcaps[k].min; 2791 2792 /* 2793 * If the field is greater than the minimum 2794 * value we can set the hwcap; 2795 */ 2796 if (mrs_field_cmp(reg, min, fields[j].shift, 2797 4, fields[j].sign) >= 0) { 2798 *cur_hwcap |= hwcaps[k].hwcap_val; 2799 } 2800 } 2801 } 2802 } 2803 } 2804 2805 static void 2806 identify_cpu_sysinit(void *dummy __unused) 2807 { 2808 struct cpu_desc *desc, *prev_desc; 2809 int cpu; 2810 bool dic, idc; 2811 2812 dic = (allow_dic != 0); 2813 idc = (allow_idc != 0); 2814 2815 prev_desc = NULL; 2816 CPU_FOREACH(cpu) { 2817 desc = get_cpu_desc(cpu); 2818 if (cpu != 0) { 2819 check_cpu_regs(cpu, desc, prev_desc); 2820 update_special_regs(cpu); 2821 } 2822 2823 if (CTR_DIC_VAL(desc->ctr) == 0) 2824 dic = false; 2825 if (CTR_IDC_VAL(desc->ctr) == 0) 2826 idc = false; 2827 prev_desc = desc; 2828 } 2829 2830 if (dic && idc) { 2831 arm64_icache_sync_range = &arm64_dic_idc_icache_sync_range; 2832 if (bootverbose) 2833 printf("Enabling DIC & IDC ICache sync\n"); 2834 } else if (idc) { 2835 arm64_icache_sync_range = &arm64_idc_aliasing_icache_sync_range; 2836 if (bootverbose) 2837 printf("Enabling IDC ICache sync\n"); 2838 } 2839 } 2840 /* 2841 * This needs to run early to ensure the kernel ID registers have been 2842 * updated for all CPUs before they are used by ifunc resolvers, etc. 2843 */ 2844 SYSINIT(identify_cpu, SI_SUB_CPU, SI_ORDER_MIDDLE, 2845 identify_cpu_sysinit, NULL); 2846 2847 static void 2848 identify_hwcaps_sysinit(void *dummy __unused) 2849 { 2850 #ifdef INVARIANTS 2851 /* Check we dont update the special registers after this point */ 2852 hwcaps_set = true; 2853 #endif 2854 2855 /* Find the values to export to userspace as AT_HWCAP and AT_HWCAP2 */ 2856 parse_cpu_features(true, &user_cpu_desc, &elf_hwcap, &elf_hwcap2); 2857 parse_cpu_features(true, &l_user_cpu_desc, &linux_elf_hwcap, 2858 &linux_elf_hwcap2); 2859 #ifdef COMPAT_FREEBSD32 2860 parse_cpu_features(false, &user_cpu_desc, &elf32_hwcap, &elf32_hwcap2); 2861 #endif 2862 2863 /* We export the CPUID registers */ 2864 elf_hwcap |= HWCAP_CPUID; 2865 linux_elf_hwcap |= HWCAP_CPUID; 2866 2867 #ifdef COMPAT_FREEBSD32 2868 /* Set the default caps and any that need to check multiple fields */ 2869 elf32_hwcap |= parse_cpu_features_hwcap32(); 2870 #endif 2871 2872 if ((elf_hwcap & HWCAP_ATOMICS) != 0) { 2873 lse_supported = true; 2874 if (bootverbose) 2875 printf("Enabling LSE atomics in the kernel\n"); 2876 } 2877 #ifdef LSE_ATOMICS 2878 if (!lse_supported) 2879 panic("CPU does not support LSE atomic instructions"); 2880 #endif 2881 2882 install_sys_handler(user_ctr_handler); 2883 install_sys_handler(user_idreg_handler); 2884 } 2885 /* 2886 * This needs to be after the APs have started as they may have errata that 2887 * means we need to mask out ID registers & that could affect hwcaps, etc. 2888 * 2889 * The errata handling runs at SI_SUB_CONFIGURE, SI_ORDER_MIDDLE + 1, so this 2890 * needs to be later than that. 2891 */ 2892 SYSINIT(identify_hwcaps, SI_SUB_CONFIGURE, SI_ORDER_MIDDLE + 2, 2893 identify_hwcaps_sysinit, NULL); 2894 2895 static void 2896 cpu_features_sysinit(void *dummy __unused) 2897 { 2898 struct sbuf sb; 2899 struct cpu_desc *desc, *prev_desc; 2900 u_int cpu; 2901 2902 prev_desc = NULL; 2903 CPU_FOREACH(cpu) { 2904 desc = get_cpu_desc(cpu); 2905 print_cpu_features(cpu, desc, prev_desc); 2906 prev_desc = desc; 2907 } 2908 2909 /* Fill in cpu_model for the hw.model sysctl */ 2910 sbuf_new(&sb, cpu_model, sizeof(cpu_model), SBUF_FIXEDLEN); 2911 print_cpu_midr(&sb, 0); 2912 2913 sbuf_finish(&sb); 2914 sbuf_delete(&sb); 2915 2916 free(cpu_desc, M_IDENTCPU); 2917 } 2918 /* Log features before APs are released and start printing to the dmesg. */ 2919 SYSINIT(cpu_features, SI_SUB_SMP - 1, SI_ORDER_ANY, cpu_features_sysinit, NULL); 2920 2921 static void 2922 tcr_set_e0pd1(void *arg __unused) 2923 { 2924 uint64_t tcr; 2925 2926 tcr = READ_SPECIALREG(tcr_el1); 2927 tcr |= TCR_E0PD1; 2928 WRITE_SPECIALREG(tcr_el1, tcr); 2929 isb(); 2930 } 2931 2932 /* Enable support for more recent architecture features */ 2933 static void 2934 cpu_feat_support(void *arg __unused) 2935 { 2936 /* 2937 * If FEAT_E0PD is supported use it to cause faults without a page 2938 * table walk if userspace tries to access kernel memory. 2939 */ 2940 if (ID_AA64MMFR2_E0PD_VAL(kern_cpu_desc.id_aa64mmfr2) != 2941 ID_AA64MMFR2_E0PD_NONE) 2942 smp_rendezvous(NULL, tcr_set_e0pd1, NULL, NULL); 2943 } 2944 SYSINIT(cpu_feat_support, SI_SUB_SMP, SI_ORDER_ANY, cpu_feat_support, NULL); 2945 2946 #ifdef COMPAT_FREEBSD32 2947 static u_long 2948 parse_cpu_features_hwcap32(void) 2949 { 2950 u_long hwcap = HWCAP32_DEFAULT; 2951 2952 if ((MVFR1_SIMDLS_VAL(user_cpu_desc.mvfr1) >= 2953 MVFR1_SIMDLS_IMPL) && 2954 (MVFR1_SIMDInt_VAL(user_cpu_desc.mvfr1) >= 2955 MVFR1_SIMDInt_IMPL) && 2956 (MVFR1_SIMDSP_VAL(user_cpu_desc.mvfr1) >= 2957 MVFR1_SIMDSP_IMPL)) 2958 hwcap |= HWCAP32_NEON; 2959 2960 return (hwcap); 2961 } 2962 #endif /* COMPAT_FREEBSD32 */ 2963 2964 static void 2965 print_register(struct sbuf *sb, const char *reg_name, uint64_t reg, 2966 void (*print_fields)(struct sbuf *, uint64_t, const void *), 2967 const void *arg) 2968 { 2969 2970 sbuf_printf(sb, "%29s = <", reg_name); 2971 2972 print_fields(sb, reg, arg); 2973 2974 sbuf_finish(sb); 2975 printf("%s>\n", sbuf_data(sb)); 2976 sbuf_clear(sb); 2977 } 2978 2979 static void 2980 print_id_fields(struct sbuf *sb, uint64_t reg, const void *arg) 2981 { 2982 const struct mrs_field *fields = arg; 2983 const struct mrs_field_value *fv; 2984 int field, i, j, printed; 2985 2986 #define SEP_STR ((printed++) == 0) ? "" : "," 2987 printed = 0; 2988 for (i = 0; fields[i].type != 0; i++) { 2989 fv = fields[i].values; 2990 2991 if (fv == NULL) 2992 goto next; 2993 2994 field = (reg & fields[i].mask) >> fields[i].shift; 2995 for (j = 0; fv[j].desc != NULL; j++) { 2996 if ((fv[j].value >> fields[i].shift) != field) 2997 continue; 2998 2999 if (fv[j].desc[0] != '\0') 3000 sbuf_printf(sb, "%s%s", SEP_STR, fv[j].desc); 3001 break; 3002 } 3003 if (fv[j].desc == NULL) 3004 sbuf_printf(sb, "%sUnknown %s(%x)", SEP_STR, 3005 fields[i].name, field); 3006 3007 next: 3008 reg &= ~(((1ul << fields[i].width) - 1) << fields[i].shift); 3009 } 3010 3011 if (reg != 0) 3012 sbuf_printf(sb, "%s%#lx", SEP_STR, reg); 3013 #undef SEP_STR 3014 } 3015 3016 static void 3017 print_id_register(struct sbuf *sb, const char *reg_name, uint64_t reg, 3018 const struct mrs_field *fields) 3019 { 3020 3021 print_register(sb, reg_name, reg, print_id_fields, fields); 3022 } 3023 3024 static void 3025 print_cpu_midr(struct sbuf *sb, u_int cpu) 3026 { 3027 const struct cpu_parts *cpu_partsp; 3028 const char *cpu_impl_name; 3029 const char *cpu_part_name; 3030 u_int midr; 3031 u_int impl_id; 3032 u_int part_id; 3033 3034 midr = pcpu_find(cpu)->pc_midr; 3035 3036 cpu_impl_name = NULL; 3037 cpu_partsp = NULL; 3038 impl_id = CPU_IMPL(midr); 3039 for (int i = 0; cpu_implementers[i].impl_name != NULL; i++) { 3040 if (impl_id == cpu_implementers[i].impl_id) { 3041 cpu_impl_name = cpu_implementers[i].impl_name; 3042 cpu_partsp = cpu_implementers[i].cpu_parts; 3043 break; 3044 } 3045 } 3046 /* Unknown implementer, so unknown part */ 3047 if (cpu_impl_name == NULL) { 3048 sbuf_printf(sb, "Unknown Implementer (midr: %08x)", midr); 3049 return; 3050 } 3051 3052 KASSERT(cpu_partsp != NULL, ("%s: No parts table for implementer %s", 3053 __func__, cpu_impl_name)); 3054 3055 cpu_part_name = NULL; 3056 part_id = CPU_PART(midr); 3057 for (int i = 0; cpu_partsp[i].part_name != NULL; i++) { 3058 if (part_id == cpu_partsp[i].part_id) { 3059 cpu_part_name = cpu_partsp[i].part_name; 3060 break; 3061 } 3062 } 3063 /* Known Implementer, Unknown part */ 3064 if (cpu_part_name == NULL) { 3065 sbuf_printf(sb, "%s Unknown CPU r%dp%d (midr: %08x)", 3066 cpu_impl_name, CPU_VAR(midr), CPU_REV(midr), midr); 3067 return; 3068 } 3069 3070 sbuf_printf(sb, "%s %s r%dp%d", cpu_impl_name, 3071 cpu_part_name, CPU_VAR(midr), CPU_REV(midr)); 3072 } 3073 3074 static void 3075 print_cpu_cache(struct cpu_desc *desc, struct sbuf *sb, uint64_t ccs, 3076 bool icache, bool unified) 3077 { 3078 size_t cache_size; 3079 size_t line_size; 3080 3081 /* LineSize is Log2(S) - 4. */ 3082 line_size = 1 << ((ccs & CCSIDR_LineSize_MASK) + 4); 3083 /* 3084 * Calculate cache size (sets * ways * line size). There are different 3085 * formats depending on the FEAT_CCIDX bit in ID_AA64MMFR2 feature 3086 * register. 3087 */ 3088 if ((desc->id_aa64mmfr2 & ID_AA64MMFR2_CCIDX_64)) 3089 cache_size = (CCSIDR_NumSets64(ccs) + 1) * 3090 (CCSIDR_Assoc64(ccs) + 1); 3091 else 3092 cache_size = (CCSIDR_NumSets(ccs) + 1) * (CCSIDR_Assoc(ccs) + 1); 3093 3094 cache_size *= line_size; 3095 sbuf_printf(sb, "%zuKB (%s)", cache_size / 1024, 3096 icache ? "instruction" : unified ? "unified" : "data"); 3097 } 3098 3099 static void 3100 print_cpu_caches(struct sbuf *sb, struct cpu_desc *desc) 3101 { 3102 /* Print out each cache combination */ 3103 uint64_t clidr; 3104 int i = 1; 3105 clidr = desc->clidr; 3106 3107 for (i = 0; (clidr & CLIDR_CTYPE_MASK) != 0; i++, clidr >>= 3) { 3108 int j = 0; 3109 int ctype_m = (clidr & CLIDR_CTYPE_MASK); 3110 3111 sbuf_printf(sb, " L%d cache: ", i + 1); 3112 if ((clidr & CLIDR_CTYPE_IO)) { 3113 print_cpu_cache(desc, sb, desc->ccsidr[i][j++], true, 3114 false); 3115 /* If there's more, add to the line. */ 3116 if ((ctype_m & ~CLIDR_CTYPE_IO) != 0) 3117 sbuf_printf(sb, ", "); 3118 } 3119 if ((ctype_m & ~CLIDR_CTYPE_IO) != 0) { 3120 print_cpu_cache(desc, sb, desc->ccsidr[i][j], false, 3121 (clidr & CLIDR_CTYPE_UNIFIED)); 3122 } 3123 sbuf_printf(sb, "\n"); 3124 3125 } 3126 sbuf_finish(sb); 3127 printf("%s", sbuf_data(sb)); 3128 } 3129 3130 static void 3131 print_cpu_features(u_int cpu, struct cpu_desc *desc, 3132 struct cpu_desc *prev_desc) 3133 { 3134 struct sbuf *sb; 3135 3136 sb = sbuf_new_auto(); 3137 sbuf_printf(sb, "CPU%3u: ", cpu); 3138 print_cpu_midr(sb, cpu); 3139 3140 sbuf_cat(sb, " affinity:"); 3141 switch(cpu_aff_levels) { 3142 default: 3143 case 4: 3144 sbuf_printf(sb, " %2d", CPU_AFF3(desc->mpidr)); 3145 /* FALLTHROUGH */ 3146 case 3: 3147 sbuf_printf(sb, " %2d", CPU_AFF2(desc->mpidr)); 3148 /* FALLTHROUGH */ 3149 case 2: 3150 sbuf_printf(sb, " %2d", CPU_AFF1(desc->mpidr)); 3151 /* FALLTHROUGH */ 3152 case 1: 3153 case 0: /* On UP this will be zero */ 3154 sbuf_printf(sb, " %2d", CPU_AFF0(desc->mpidr)); 3155 break; 3156 } 3157 sbuf_finish(sb); 3158 printf("%s\n", sbuf_data(sb)); 3159 sbuf_clear(sb); 3160 3161 /* 3162 * There is a hardware errata where, if one CPU is performing a TLB 3163 * invalidation while another is performing a store-exclusive the 3164 * store-exclusive may return the wrong status. A workaround seems 3165 * to be to use an IPI to invalidate on each CPU, however given the 3166 * limited number of affected units (pass 1.1 is the evaluation 3167 * hardware revision), and the lack of information from Cavium 3168 * this has not been implemented. 3169 * 3170 * At the time of writing this the only information is from: 3171 * https://lkml.org/lkml/2016/8/4/722 3172 */ 3173 /* 3174 * XXX: CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1 on its own also 3175 * triggers on pass 2.0+. 3176 */ 3177 if (cpu == 0 && CPU_VAR(PCPU_GET(midr)) == 0 && 3178 CPU_MATCH_ERRATA_CAVIUM_THUNDERX_1_1) 3179 printf("WARNING: ThunderX Pass 1.1 detected.\nThis has known " 3180 "hardware bugs that may cause the incorrect operation of " 3181 "atomic operations.\n"); 3182 3183 #define SHOULD_PRINT_REG(_reg) \ 3184 (prev_desc == NULL || desc->_reg != prev_desc->_reg) 3185 3186 /* Cache Type Register */ 3187 if (SHOULD_PRINT_REG(ctr)) 3188 print_id_register(sb, "Cache Type", desc->ctr, ctr_fields); 3189 3190 /* AArch64 Instruction Set Attribute Register 0 */ 3191 if (SHOULD_PRINT_REG(id_aa64isar0)) 3192 print_id_register(sb, "Instruction Set Attributes 0", 3193 desc->id_aa64isar0, id_aa64isar0_fields); 3194 3195 /* AArch64 Instruction Set Attribute Register 1 */ 3196 if (SHOULD_PRINT_REG(id_aa64isar1)) 3197 print_id_register(sb, "Instruction Set Attributes 1", 3198 desc->id_aa64isar1, id_aa64isar1_fields); 3199 3200 /* AArch64 Instruction Set Attribute Register 2 */ 3201 if (SHOULD_PRINT_REG(id_aa64isar2)) 3202 print_id_register(sb, "Instruction Set Attributes 2", 3203 desc->id_aa64isar2, id_aa64isar2_fields); 3204 3205 /* AArch64 Processor Feature Register 0 */ 3206 if (SHOULD_PRINT_REG(id_aa64pfr0)) 3207 print_id_register(sb, "Processor Features 0", 3208 desc->id_aa64pfr0, id_aa64pfr0_fields); 3209 3210 /* AArch64 Processor Feature Register 1 */ 3211 if (SHOULD_PRINT_REG(id_aa64pfr1)) 3212 print_id_register(sb, "Processor Features 1", 3213 desc->id_aa64pfr1, id_aa64pfr1_fields); 3214 3215 /* AArch64 Processor Feature Register 2 */ 3216 if (SHOULD_PRINT_REG(id_aa64pfr2)) 3217 print_id_register(sb, "Processor Features 2", 3218 desc->id_aa64pfr2, id_aa64pfr2_fields); 3219 3220 /* AArch64 Memory Model Feature Register 0 */ 3221 if (SHOULD_PRINT_REG(id_aa64mmfr0)) 3222 print_id_register(sb, "Memory Model Features 0", 3223 desc->id_aa64mmfr0, id_aa64mmfr0_fields); 3224 3225 /* AArch64 Memory Model Feature Register 1 */ 3226 if (SHOULD_PRINT_REG(id_aa64mmfr1)) 3227 print_id_register(sb, "Memory Model Features 1", 3228 desc->id_aa64mmfr1, id_aa64mmfr1_fields); 3229 3230 /* AArch64 Memory Model Feature Register 2 */ 3231 if (SHOULD_PRINT_REG(id_aa64mmfr2)) 3232 print_id_register(sb, "Memory Model Features 2", 3233 desc->id_aa64mmfr2, id_aa64mmfr2_fields); 3234 3235 /* AArch64 Memory Model Feature Register 3 */ 3236 if (SHOULD_PRINT_REG(id_aa64mmfr3)) 3237 print_id_register(sb, "Memory Model Features 3", 3238 desc->id_aa64mmfr3, id_aa64mmfr3_fields); 3239 3240 /* AArch64 Memory Model Feature Register 4 */ 3241 if (SHOULD_PRINT_REG(id_aa64mmfr4)) 3242 print_id_register(sb, "Memory Model Features 4", 3243 desc->id_aa64mmfr4, id_aa64mmfr4_fields); 3244 3245 /* AArch64 Debug Feature Register 0 */ 3246 if (SHOULD_PRINT_REG(id_aa64dfr0)) 3247 print_id_register(sb, "Debug Features 0", 3248 desc->id_aa64dfr0, id_aa64dfr0_fields); 3249 3250 /* AArch64 Memory Model Feature Register 1 */ 3251 if (SHOULD_PRINT_REG(id_aa64dfr1)) 3252 print_id_register(sb, "Debug Features 1", 3253 desc->id_aa64dfr1, id_aa64dfr1_fields); 3254 3255 /* AArch64 Auxiliary Feature Register 0 */ 3256 if (SHOULD_PRINT_REG(id_aa64afr0)) 3257 print_id_register(sb, "Auxiliary Features 0", 3258 desc->id_aa64afr0, id_aa64afr0_fields); 3259 3260 /* AArch64 Auxiliary Feature Register 1 */ 3261 if (SHOULD_PRINT_REG(id_aa64afr1)) 3262 print_id_register(sb, "Auxiliary Features 1", 3263 desc->id_aa64afr1, id_aa64afr1_fields); 3264 3265 /* AArch64 SVE Feature Register 0 */ 3266 if (desc->have_sve) { 3267 if (SHOULD_PRINT_REG(id_aa64zfr0) || 3268 !prev_desc->have_sve) { 3269 print_id_register(sb, "SVE Features 0", 3270 desc->id_aa64zfr0, id_aa64zfr0_fields); 3271 } 3272 } 3273 3274 #ifdef COMPAT_FREEBSD32 3275 /* AArch32 Instruction Set Attribute Register 5 */ 3276 if (SHOULD_PRINT_REG(id_isar5)) 3277 print_id_register(sb, "AArch32 Instruction Set Attributes 5", 3278 desc->id_isar5, id_isar5_fields); 3279 3280 /* AArch32 Media and VFP Feature Register 0 */ 3281 if (SHOULD_PRINT_REG(mvfr0)) 3282 print_id_register(sb, "AArch32 Media and VFP Features 0", 3283 desc->mvfr0, mvfr0_fields); 3284 3285 /* AArch32 Media and VFP Feature Register 1 */ 3286 if (SHOULD_PRINT_REG(mvfr1)) 3287 print_id_register(sb, "AArch32 Media and VFP Features 1", 3288 desc->mvfr1, mvfr1_fields); 3289 #endif 3290 if (bootverbose) 3291 print_cpu_caches(sb, desc); 3292 3293 sbuf_delete(sb); 3294 sb = NULL; 3295 #undef SHOULD_PRINT_REG 3296 #undef SEP_STR 3297 } 3298 3299 void 3300 identify_cache(uint64_t ctr) 3301 { 3302 3303 /* Identify the L1 cache type */ 3304 switch (CTR_L1IP_VAL(ctr)) { 3305 case CTR_L1IP_PIPT: 3306 break; 3307 default: 3308 case CTR_L1IP_VIPT: 3309 icache_aliasing = true; 3310 break; 3311 } 3312 3313 if (dcache_line_size == 0) { 3314 KASSERT(icache_line_size == 0, ("%s: i-cacheline size set: %ld", 3315 __func__, icache_line_size)); 3316 3317 /* Get the D cache line size */ 3318 dcache_line_size = CTR_DLINE_SIZE(ctr); 3319 /* And the same for the I cache */ 3320 icache_line_size = CTR_ILINE_SIZE(ctr); 3321 3322 idcache_line_size = MIN(dcache_line_size, icache_line_size); 3323 } 3324 3325 if (dcache_line_size != CTR_DLINE_SIZE(ctr)) { 3326 printf("WARNING: D-cacheline size mismatch %ld != %d\n", 3327 dcache_line_size, CTR_DLINE_SIZE(ctr)); 3328 } 3329 3330 if (icache_line_size != CTR_ILINE_SIZE(ctr)) { 3331 printf("WARNING: I-cacheline size mismatch %ld != %d\n", 3332 icache_line_size, CTR_ILINE_SIZE(ctr)); 3333 } 3334 } 3335 3336 void 3337 identify_cpu(u_int cpu) 3338 { 3339 struct cpu_desc *desc; 3340 uint64_t clidr; 3341 3342 desc = get_cpu_desc(cpu); 3343 /* Save affinity for current CPU */ 3344 desc->mpidr = get_mpidr(); 3345 CPU_AFFINITY(cpu) = desc->mpidr & CPU_AFF_MASK; 3346 3347 desc->ctr = READ_SPECIALREG(ctr_el0); 3348 desc->id_aa64dfr0 = READ_SPECIALREG(ID_AA64DFR0_EL1_REG); 3349 desc->id_aa64dfr1 = READ_SPECIALREG(ID_AA64DFR1_EL1_REG); 3350 desc->id_aa64isar0 = READ_SPECIALREG(ID_AA64ISAR0_EL1_REG); 3351 desc->id_aa64isar1 = READ_SPECIALREG(ID_AA64ISAR1_EL1_REG); 3352 desc->id_aa64isar2 = READ_SPECIALREG(ID_AA64ISAR2_EL1_REG); 3353 desc->id_aa64mmfr0 = READ_SPECIALREG(ID_AA64MMFR0_EL1_REG); 3354 desc->id_aa64mmfr1 = READ_SPECIALREG(ID_AA64MMFR1_EL1_REG); 3355 desc->id_aa64mmfr2 = READ_SPECIALREG(ID_AA64MMFR2_EL1_REG); 3356 desc->id_aa64mmfr3 = READ_SPECIALREG(ID_AA64MMFR3_EL1_REG); 3357 desc->id_aa64mmfr4 = READ_SPECIALREG(ID_AA64MMFR4_EL1_REG); 3358 desc->id_aa64pfr0 = READ_SPECIALREG(ID_AA64PFR0_EL1_REG); 3359 desc->id_aa64pfr1 = READ_SPECIALREG(ID_AA64PFR1_EL1_REG); 3360 desc->id_aa64pfr2 = READ_SPECIALREG(ID_AA64PFR2_EL1_REG); 3361 3362 /* 3363 * ID_AA64ZFR0_EL1 is only valid when at least one of: 3364 * - ID_AA64PFR0_EL1.SVE is non-zero 3365 * - ID_AA64PFR1_EL1.SME is non-zero 3366 * In other cases it is zero, but still safe to read 3367 */ 3368 desc->have_sve = 3369 (ID_AA64PFR0_SVE_VAL(desc->id_aa64pfr0) != 0); 3370 desc->id_aa64zfr0 = READ_SPECIALREG(ID_AA64ZFR0_EL1_REG); 3371 3372 desc->clidr = READ_SPECIALREG(clidr_el1); 3373 3374 clidr = desc->clidr; 3375 3376 for (int i = 0; (clidr & CLIDR_CTYPE_MASK) != 0; i++, clidr >>= 3) { 3377 int j = 0; 3378 if ((clidr & CLIDR_CTYPE_IO)) { 3379 WRITE_SPECIALREG(csselr_el1, 3380 CSSELR_Level(i) | CSSELR_InD_IC); 3381 desc->ccsidr[i][j++] = 3382 READ_SPECIALREG(ccsidr_el1); 3383 } 3384 if ((clidr & ~CLIDR_CTYPE_IO) == 0) 3385 continue; 3386 WRITE_SPECIALREG(csselr_el1, CSSELR_Level(i)); 3387 desc->ccsidr[i][j] = READ_SPECIALREG(ccsidr_el1); 3388 } 3389 3390 #ifdef COMPAT_FREEBSD32 3391 /* Only read aarch32 SRs if EL0-32 is available */ 3392 if (ID_AA64PFR0_EL0_VAL(desc->id_aa64pfr0) == ID_AA64PFR0_EL0_64_32) { 3393 desc->id_isar5 = READ_SPECIALREG(id_isar5_el1); 3394 desc->mvfr0 = READ_SPECIALREG(mvfr0_el1); 3395 desc->mvfr1 = READ_SPECIALREG(mvfr1_el1); 3396 } 3397 #endif 3398 } 3399 3400 static void 3401 check_cpu_regs(u_int cpu, struct cpu_desc *desc, struct cpu_desc *prev_desc) 3402 { 3403 switch (cpu_aff_levels) { 3404 case 0: 3405 if (CPU_AFF0(desc->mpidr) != CPU_AFF0(prev_desc->mpidr)) 3406 cpu_aff_levels = 1; 3407 /* FALLTHROUGH */ 3408 case 1: 3409 if (CPU_AFF1(desc->mpidr) != CPU_AFF1(prev_desc->mpidr)) 3410 cpu_aff_levels = 2; 3411 /* FALLTHROUGH */ 3412 case 2: 3413 if (CPU_AFF2(desc->mpidr) != CPU_AFF2(prev_desc->mpidr)) 3414 cpu_aff_levels = 3; 3415 /* FALLTHROUGH */ 3416 case 3: 3417 if (CPU_AFF3(desc->mpidr) != CPU_AFF3(prev_desc->mpidr)) 3418 cpu_aff_levels = 4; 3419 break; 3420 } 3421 3422 if (desc->ctr != prev_desc->ctr) { 3423 /* 3424 * If the cache is different on different cores we should 3425 * emulate for userspace to provide a uniform value 3426 */ 3427 emulate_ctr = true; 3428 3429 /* 3430 * If the cache type register is different we may 3431 * have a different l1 cache type. 3432 */ 3433 identify_cache(desc->ctr); 3434 } 3435 } 3436