Lines Matching +full:secondary +full:- +full:boot +full:- +full:reg
1 // SPDX-License-Identifier: GPL-2.0-only
9 * there's a little bit of over-abstraction that tends to obscure what's going
14 * user-visible instructions are available only on a subset of the available
16 * boot CPU and comparing these with the feature registers of each secondary
18 * snapshot state to indicate the lowest-common denominator of the feature,
31 * - Mismatched features are *always* sanitised to a "safe" value, which
34 * - A mismatched feature marked with FTR_STRICT will cause a "SANITY CHECK"
38 * - Features marked as FTR_VISIBLE have their sanitised value visible to
43 * - A "feature" is typically a 4-bit register field. A "capability" is the
44 * high-level description derived from the sanitised field value.
46 * - Read the Arm ARM (DDI 0487F.a) section D13.1.3 ("Principles of the ID
50 * - KVM exposes its own view of the feature registers to guest operating
57 * - If the arm64_ftr_bits[] for a register has a missing field, then this
120 * Permit PER_LINUX32 and execve() of 32-bit binaries even if not all CPUs
127 * seen at least one CPU capable of 32-bit EL0.
132 * Mask of CPUs supporting 32-bit EL0.
139 /* file-wide pr_fmt adds "CPU features: " prefix */ in dump_cpu_features()
143 #define __ARM64_MAX_POSITIVE(reg, field) \ argument
144 ((reg##_##field##_SIGNED ? \
145 BIT(reg##_##field##_WIDTH - 1) : \
146 BIT(reg##_##field##_WIDTH)) - 1)
148 #define __ARM64_MIN_NEGATIVE(reg, field) BIT(reg##_##field##_WIDTH - 1) argument
150 #define __ARM64_CPUID_FIELDS(reg, field, min_value, max_value) \ argument
151 .sys_reg = SYS_##reg, \
152 .field_pos = reg##_##field##_SHIFT, \
153 .field_width = reg##_##field##_WIDTH, \
154 .sign = reg##_##field##_SIGNED, \
160 * an implicit maximum that depends on the sign-ess of the field.
165 #define ARM64_CPUID_FIELDS(reg, field, min_value) \ argument
166 __ARM64_CPUID_FIELDS(reg, field, \
167 SYS_FIELD_VALUE(reg, field, min_value), \
168 __ARM64_MAX_POSITIVE(reg, field))
173 * matching a non-implemented property.
175 #define ARM64_CPUID_FIELDS_NEG(reg, field, max_value) \ argument
176 __ARM64_CPUID_FIELDS(reg, field, \
177 __ARM64_MIN_NEGATIVE(reg, field), \
178 SYS_FIELD_VALUE(reg, field, max_value))
390 * Page size not being supported at Stage-2 is not fatal. You
395 * advertises a given granule size at Stage-2 (value 2) on some
396 * vCPUs, and uses the fallback to Stage-1 (value 0) for other
408 * We already refuse to boot CPUs that don't support our configured
412 * along with it and treat them as non-strict.
489 * Linux can handle differing I-cache policies. Userspace JITs will
491 * If we have differing I-cache policies, report it as the weakest - VIPT.
691 * id_isar[1-3], id_mmfr[1-3]
717 .reg = &(struct arm64_ftr_reg){ \
743 struct arm64_ftr_reg *reg; member
820 return (int)(unsigned long)id - (int)((const struct __ftr_reg_entry *)regp)->sys_id; in search_cmp_ftr_reg()
824 * get_arm64_ftr_reg_nowarn - Looks up a feature register entry using
829 * returns - Upon success, matching ftr_reg entry for id.
830 * - NULL on failure. It is upto the caller to decide
843 return ret->reg; in get_arm64_ftr_reg_nowarn()
848 * get_arm64_ftr_reg - Looks up a feature register entry using
851 * returns - Upon success, matching ftr_reg entry for id.
852 * - NULL on failure but with an WARN_ON().
856 struct arm64_ftr_reg *reg; in get_arm64_ftr_reg() local
858 reg = get_arm64_ftr_reg_nowarn(sys_id); in get_arm64_ftr_reg()
861 * Requesting a non-existent register search is an error. Warn in get_arm64_ftr_reg()
864 WARN_ON(!reg); in get_arm64_ftr_reg()
865 return reg; in get_arm64_ftr_reg()
868 static u64 arm64_ftr_set_value(const struct arm64_ftr_bits *ftrp, s64 reg, in arm64_ftr_set_value() argument
873 reg &= ~mask; in arm64_ftr_set_value()
874 reg |= (ftr_val << ftrp->shift) & mask; in arm64_ftr_set_value()
875 return reg; in arm64_ftr_set_value()
883 switch (ftrp->type) { in arm64_ftr_safe_value()
885 ret = ftrp->safe_val; in arm64_ftr_safe_value()
909 const struct arm64_ftr_reg *ftr_reg = arm64_ftr_regs[i].reg; in sort_ftr_regs()
910 const struct arm64_ftr_bits *ftr_bits = ftr_reg->ftr_bits; in sort_ftr_regs()
917 for (; ftr_bits->width != 0; ftr_bits++, j++) { in sort_ftr_regs()
918 unsigned int width = ftr_reg->ftr_bits[j].width; in sort_ftr_regs()
919 unsigned int shift = ftr_reg->ftr_bits[j].shift; in sort_ftr_regs()
924 ftr_reg->name, shift); in sort_ftr_regs()
933 prev_shift = ftr_reg->ftr_bits[j - 1].shift; in sort_ftr_regs()
936 ftr_reg->name, shift); in sort_ftr_regs()
950 BUG_ON(arm64_ftr_regs[i].sys_id <= arm64_ftr_regs[i - 1].sys_id); in sort_ftr_regs()
955 * Initialise the CPU feature register from Boot CPU values.
958 * RES0 for the system-wide value, and must strictly match.
968 struct arm64_ftr_reg *reg = get_arm64_ftr_reg(sys_reg); in init_cpu_ftr_reg() local
970 if (!reg) in init_cpu_ftr_reg()
973 for (ftrp = reg->ftr_bits; ftrp->width; ftrp++) { in init_cpu_ftr_reg()
976 s64 ftr_ovr = arm64_ftr_value(ftrp, reg->override->val); in init_cpu_ftr_reg()
978 if ((ftr_mask & reg->override->mask) == ftr_mask) { in init_cpu_ftr_reg()
984 reg->override->mask &= ~ftr_mask; in init_cpu_ftr_reg()
985 reg->override->val &= ~ftr_mask; in init_cpu_ftr_reg()
999 reg->name, in init_cpu_ftr_reg()
1000 ftrp->shift + ftrp->width - 1, in init_cpu_ftr_reg()
1001 ftrp->shift, str, in init_cpu_ftr_reg()
1002 tmp & (BIT(ftrp->width) - 1)); in init_cpu_ftr_reg()
1003 } else if ((ftr_mask & reg->override->val) == ftr_mask) { in init_cpu_ftr_reg()
1004 reg->override->val &= ~ftr_mask; in init_cpu_ftr_reg()
1006 reg->name, in init_cpu_ftr_reg()
1007 ftrp->shift + ftrp->width - 1, in init_cpu_ftr_reg()
1008 ftrp->shift); in init_cpu_ftr_reg()
1014 if (!ftrp->strict) in init_cpu_ftr_reg()
1016 if (ftrp->visible) in init_cpu_ftr_reg()
1019 reg->user_val = arm64_ftr_set_value(ftrp, in init_cpu_ftr_reg()
1020 reg->user_val, in init_cpu_ftr_reg()
1021 ftrp->safe_val); in init_cpu_ftr_reg()
1026 reg->sys_val = val; in init_cpu_ftr_reg()
1027 reg->strict_mask = strict_mask; in init_cpu_ftr_reg()
1028 reg->user_mask = user_mask; in init_cpu_ftr_reg()
1037 for (; caps->matches; caps++) { in init_cpucap_indirect_list_from_array()
1038 if (WARN(caps->capability >= ARM64_NCAPS, in init_cpucap_indirect_list_from_array()
1039 "Invalid capability %d\n", caps->capability)) in init_cpucap_indirect_list_from_array()
1041 if (WARN(cpucap_ptrs[caps->capability], in init_cpucap_indirect_list_from_array()
1043 caps->capability)) in init_cpucap_indirect_list_from_array()
1045 cpucap_ptrs[caps->capability] = caps; in init_cpucap_indirect_list_from_array()
1059 init_cpu_ftr_reg(SYS_ID_DFR0_EL1, info->reg_id_dfr0); in init_32bit_cpu_features()
1060 init_cpu_ftr_reg(SYS_ID_DFR1_EL1, info->reg_id_dfr1); in init_32bit_cpu_features()
1061 init_cpu_ftr_reg(SYS_ID_ISAR0_EL1, info->reg_id_isar0); in init_32bit_cpu_features()
1062 init_cpu_ftr_reg(SYS_ID_ISAR1_EL1, info->reg_id_isar1); in init_32bit_cpu_features()
1063 init_cpu_ftr_reg(SYS_ID_ISAR2_EL1, info->reg_id_isar2); in init_32bit_cpu_features()
1064 init_cpu_ftr_reg(SYS_ID_ISAR3_EL1, info->reg_id_isar3); in init_32bit_cpu_features()
1065 init_cpu_ftr_reg(SYS_ID_ISAR4_EL1, info->reg_id_isar4); in init_32bit_cpu_features()
1066 init_cpu_ftr_reg(SYS_ID_ISAR5_EL1, info->reg_id_isar5); in init_32bit_cpu_features()
1067 init_cpu_ftr_reg(SYS_ID_ISAR6_EL1, info->reg_id_isar6); in init_32bit_cpu_features()
1068 init_cpu_ftr_reg(SYS_ID_MMFR0_EL1, info->reg_id_mmfr0); in init_32bit_cpu_features()
1069 init_cpu_ftr_reg(SYS_ID_MMFR1_EL1, info->reg_id_mmfr1); in init_32bit_cpu_features()
1070 init_cpu_ftr_reg(SYS_ID_MMFR2_EL1, info->reg_id_mmfr2); in init_32bit_cpu_features()
1071 init_cpu_ftr_reg(SYS_ID_MMFR3_EL1, info->reg_id_mmfr3); in init_32bit_cpu_features()
1072 init_cpu_ftr_reg(SYS_ID_MMFR4_EL1, info->reg_id_mmfr4); in init_32bit_cpu_features()
1073 init_cpu_ftr_reg(SYS_ID_MMFR5_EL1, info->reg_id_mmfr5); in init_32bit_cpu_features()
1074 init_cpu_ftr_reg(SYS_ID_PFR0_EL1, info->reg_id_pfr0); in init_32bit_cpu_features()
1075 init_cpu_ftr_reg(SYS_ID_PFR1_EL1, info->reg_id_pfr1); in init_32bit_cpu_features()
1076 init_cpu_ftr_reg(SYS_ID_PFR2_EL1, info->reg_id_pfr2); in init_32bit_cpu_features()
1077 init_cpu_ftr_reg(SYS_MVFR0_EL1, info->reg_mvfr0); in init_32bit_cpu_features()
1078 init_cpu_ftr_reg(SYS_MVFR1_EL1, info->reg_mvfr1); in init_32bit_cpu_features()
1079 init_cpu_ftr_reg(SYS_MVFR2_EL1, info->reg_mvfr2); in init_32bit_cpu_features()
1102 np = of_find_compatible_node(NULL, NULL, "arm,gic-v3"); in detect_system_supports_pseudo_nmi()
1103 if (np && of_property_read_bool(np, "mediatek,broken-save-restore-fw")) { in detect_system_supports_pseudo_nmi()
1104 pr_info("Pseudo-NMI disabled due to MediaTek Chromebook GICR save problem\n"); in detect_system_supports_pseudo_nmi()
1118 init_cpu_ftr_reg(SYS_CTR_EL0, info->reg_ctr); in init_cpu_features()
1119 init_cpu_ftr_reg(SYS_DCZID_EL0, info->reg_dczid); in init_cpu_features()
1120 init_cpu_ftr_reg(SYS_CNTFRQ_EL0, info->reg_cntfrq); in init_cpu_features()
1121 init_cpu_ftr_reg(SYS_ID_AA64DFR0_EL1, info->reg_id_aa64dfr0); in init_cpu_features()
1122 init_cpu_ftr_reg(SYS_ID_AA64DFR1_EL1, info->reg_id_aa64dfr1); in init_cpu_features()
1123 init_cpu_ftr_reg(SYS_ID_AA64ISAR0_EL1, info->reg_id_aa64isar0); in init_cpu_features()
1124 init_cpu_ftr_reg(SYS_ID_AA64ISAR1_EL1, info->reg_id_aa64isar1); in init_cpu_features()
1125 init_cpu_ftr_reg(SYS_ID_AA64ISAR2_EL1, info->reg_id_aa64isar2); in init_cpu_features()
1126 init_cpu_ftr_reg(SYS_ID_AA64ISAR3_EL1, info->reg_id_aa64isar3); in init_cpu_features()
1127 init_cpu_ftr_reg(SYS_ID_AA64MMFR0_EL1, info->reg_id_aa64mmfr0); in init_cpu_features()
1128 init_cpu_ftr_reg(SYS_ID_AA64MMFR1_EL1, info->reg_id_aa64mmfr1); in init_cpu_features()
1129 init_cpu_ftr_reg(SYS_ID_AA64MMFR2_EL1, info->reg_id_aa64mmfr2); in init_cpu_features()
1130 init_cpu_ftr_reg(SYS_ID_AA64MMFR3_EL1, info->reg_id_aa64mmfr3); in init_cpu_features()
1131 init_cpu_ftr_reg(SYS_ID_AA64MMFR4_EL1, info->reg_id_aa64mmfr4); in init_cpu_features()
1132 init_cpu_ftr_reg(SYS_ID_AA64PFR0_EL1, info->reg_id_aa64pfr0); in init_cpu_features()
1133 init_cpu_ftr_reg(SYS_ID_AA64PFR1_EL1, info->reg_id_aa64pfr1); in init_cpu_features()
1134 init_cpu_ftr_reg(SYS_ID_AA64PFR2_EL1, info->reg_id_aa64pfr2); in init_cpu_features()
1135 init_cpu_ftr_reg(SYS_ID_AA64ZFR0_EL1, info->reg_id_aa64zfr0); in init_cpu_features()
1136 init_cpu_ftr_reg(SYS_ID_AA64SMFR0_EL1, info->reg_id_aa64smfr0); in init_cpu_features()
1137 init_cpu_ftr_reg(SYS_ID_AA64FPFR0_EL1, info->reg_id_aa64fpfr0); in init_cpu_features()
1139 if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) in init_cpu_features()
1140 init_32bit_cpu_features(&info->aarch32); in init_cpu_features()
1160 info->reg_smidr = read_cpuid(SMIDR_EL1) & ~SMIDR_EL1_SMPS; in init_cpu_features()
1166 if (id_aa64pfr1_mte(info->reg_id_aa64pfr1)) in init_cpu_features()
1167 init_cpu_ftr_reg(SYS_GMID_EL1, info->reg_gmid); in init_cpu_features()
1170 static void update_cpu_ftr_reg(struct arm64_ftr_reg *reg, u64 new) in update_cpu_ftr_reg() argument
1174 for (ftrp = reg->ftr_bits; ftrp->width; ftrp++) { in update_cpu_ftr_reg()
1175 s64 ftr_cur = arm64_ftr_value(ftrp, reg->sys_val); in update_cpu_ftr_reg()
1182 reg->sys_val = arm64_ftr_set_value(ftrp, reg->sys_val, ftr_new); in update_cpu_ftr_reg()
1187 static int check_update_ftr_reg(u32 sys_id, int cpu, u64 val, u64 boot) in check_update_ftr_reg() argument
1195 if ((boot & regp->strict_mask) == (val & regp->strict_mask)) in check_update_ftr_reg()
1197 pr_warn("SANITY CHECK: Unexpected variation in %s. Boot CPU: %#016llx, CPU%d: %#016llx\n", in check_update_ftr_reg()
1198 regp->name, boot, cpu, val); in check_update_ftr_reg()
1210 for (ftrp = regp->ftr_bits; ftrp->width; ftrp++) { in relax_cpu_ftr_reg()
1211 if (ftrp->shift == field) { in relax_cpu_ftr_reg()
1212 regp->strict_mask &= ~arm64_ftr_mask(ftrp); in relax_cpu_ftr_reg()
1218 WARN_ON(!ftrp->width); in relax_cpu_ftr_reg()
1222 struct cpuinfo_arm64 *boot) in lazy_init_32bit_cpu_features() argument
1229 if (id_aa64pfr0_32bit_el0(boot->reg_id_aa64pfr0)) in lazy_init_32bit_cpu_features()
1232 boot->aarch32 = info->aarch32; in lazy_init_32bit_cpu_features()
1233 init_32bit_cpu_features(&boot->aarch32); in lazy_init_32bit_cpu_features()
1238 struct cpuinfo_32bit *boot) in update_32bit_cpu_features() argument
1245 * EL1-dependent register fields to avoid spurious sanity check fails. in update_32bit_cpu_features()
1257 info->reg_id_dfr0, boot->reg_id_dfr0); in update_32bit_cpu_features()
1259 info->reg_id_dfr1, boot->reg_id_dfr1); in update_32bit_cpu_features()
1261 info->reg_id_isar0, boot->reg_id_isar0); in update_32bit_cpu_features()
1263 info->reg_id_isar1, boot->reg_id_isar1); in update_32bit_cpu_features()
1265 info->reg_id_isar2, boot->reg_id_isar2); in update_32bit_cpu_features()
1267 info->reg_id_isar3, boot->reg_id_isar3); in update_32bit_cpu_features()
1269 info->reg_id_isar4, boot->reg_id_isar4); in update_32bit_cpu_features()
1271 info->reg_id_isar5, boot->reg_id_isar5); in update_32bit_cpu_features()
1273 info->reg_id_isar6, boot->reg_id_isar6); in update_32bit_cpu_features()
1281 info->reg_id_mmfr0, boot->reg_id_mmfr0); in update_32bit_cpu_features()
1283 info->reg_id_mmfr1, boot->reg_id_mmfr1); in update_32bit_cpu_features()
1285 info->reg_id_mmfr2, boot->reg_id_mmfr2); in update_32bit_cpu_features()
1287 info->reg_id_mmfr3, boot->reg_id_mmfr3); in update_32bit_cpu_features()
1289 info->reg_id_mmfr4, boot->reg_id_mmfr4); in update_32bit_cpu_features()
1291 info->reg_id_mmfr5, boot->reg_id_mmfr5); in update_32bit_cpu_features()
1293 info->reg_id_pfr0, boot->reg_id_pfr0); in update_32bit_cpu_features()
1295 info->reg_id_pfr1, boot->reg_id_pfr1); in update_32bit_cpu_features()
1297 info->reg_id_pfr2, boot->reg_id_pfr2); in update_32bit_cpu_features()
1299 info->reg_mvfr0, boot->reg_mvfr0); in update_32bit_cpu_features()
1301 info->reg_mvfr1, boot->reg_mvfr1); in update_32bit_cpu_features()
1303 info->reg_mvfr2, boot->reg_mvfr2); in update_32bit_cpu_features()
1310 * non-boot CPU. Also performs SANITY checks to make sure that there
1311 * aren't any insane variations from that of the boot CPU.
1315 struct cpuinfo_arm64 *boot) in update_cpu_features() argument
1320 * The kernel can handle differing I-cache policies, but otherwise in update_cpu_features()
1325 info->reg_ctr, boot->reg_ctr); in update_cpu_features()
1333 info->reg_dczid, boot->reg_dczid); in update_cpu_features()
1337 info->reg_cntfrq, boot->reg_cntfrq); in update_cpu_features()
1340 * The kernel uses self-hosted debug features and expects CPUs to in update_cpu_features()
1346 info->reg_id_aa64dfr0, boot->reg_id_aa64dfr0); in update_cpu_features()
1348 info->reg_id_aa64dfr1, boot->reg_id_aa64dfr1); in update_cpu_features()
1350 * Even in big.LITTLE, processors should be identical instruction-set in update_cpu_features()
1354 info->reg_id_aa64isar0, boot->reg_id_aa64isar0); in update_cpu_features()
1356 info->reg_id_aa64isar1, boot->reg_id_aa64isar1); in update_cpu_features()
1358 info->reg_id_aa64isar2, boot->reg_id_aa64isar2); in update_cpu_features()
1360 info->reg_id_aa64isar3, boot->reg_id_aa64isar3); in update_cpu_features()
1368 info->reg_id_aa64mmfr0, boot->reg_id_aa64mmfr0); in update_cpu_features()
1370 info->reg_id_aa64mmfr1, boot->reg_id_aa64mmfr1); in update_cpu_features()
1372 info->reg_id_aa64mmfr2, boot->reg_id_aa64mmfr2); in update_cpu_features()
1374 info->reg_id_aa64mmfr3, boot->reg_id_aa64mmfr3); in update_cpu_features()
1377 info->reg_id_aa64pfr0, boot->reg_id_aa64pfr0); in update_cpu_features()
1379 info->reg_id_aa64pfr1, boot->reg_id_aa64pfr1); in update_cpu_features()
1381 info->reg_id_aa64pfr2, boot->reg_id_aa64pfr2); in update_cpu_features()
1384 info->reg_id_aa64zfr0, boot->reg_id_aa64zfr0); in update_cpu_features()
1387 info->reg_id_aa64smfr0, boot->reg_id_aa64smfr0); in update_cpu_features()
1390 info->reg_id_aa64fpfr0, boot->reg_id_aa64fpfr0); in update_cpu_features()
1413 info->reg_smidr = read_cpuid(SMIDR_EL1) & ~SMIDR_EL1_SMPS; in update_cpu_features()
1428 id_aa64pfr1_mte(info->reg_id_aa64pfr1)) { in update_cpu_features()
1430 info->reg_gmid, boot->reg_gmid); in update_cpu_features()
1441 if (id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0)) { in update_cpu_features()
1442 lazy_init_32bit_cpu_features(info, boot); in update_cpu_features()
1443 taint |= update_32bit_cpu_features(cpu, &info->aarch32, in update_cpu_features()
1444 &boot->aarch32); in update_cpu_features()
1463 return regp->sys_val; in read_sanitised_ftr_reg()
1471 * __read_sysreg_by_encoding() - Used by a STARTING cpu before cpuinfo is populated.
1531 val &= ~regp->override->mask; in __read_sysreg_by_encoding()
1532 val |= (regp->override->val & regp->override->mask); in __read_sysreg_by_encoding()
1538 #include <linux/irqchip/arm-gic-v3.h>
1547 feature_matches(u64 reg, const struct arm64_cpu_capabilities *entry) in feature_matches() argument
1552 val = cpuid_feature_extract_field_width(reg, entry->field_pos, in feature_matches()
1553 entry->field_width, in feature_matches()
1554 entry->sign); in feature_matches()
1556 tmp = entry->min_field_value; in feature_matches()
1557 tmp <<= entry->field_pos; in feature_matches()
1559 min = cpuid_feature_extract_field_width(tmp, entry->field_pos, in feature_matches()
1560 entry->field_width, in feature_matches()
1561 entry->sign); in feature_matches()
1563 tmp = entry->max_field_value; in feature_matches()
1564 tmp <<= entry->field_pos; in feature_matches()
1566 max = cpuid_feature_extract_field_width(tmp, entry->field_pos, in feature_matches()
1567 entry->field_width, in feature_matches()
1568 entry->sign); in feature_matches()
1578 return read_sanitised_ftr_reg(entry->sys_reg); in read_scoped_sysreg()
1580 return __read_sysreg_by_encoding(entry->sys_reg); in read_scoped_sysreg()
1590 regp = get_arm64_ftr_reg(entry->sys_reg); in has_user_cpuid_feature()
1594 mask = cpuid_feature_extract_unsigned_field_width(regp->user_mask, in has_user_cpuid_feature()
1595 entry->field_pos, in has_user_cpuid_feature()
1596 entry->field_width); in has_user_cpuid_feature()
1660 pr_info("detected: 32-bit EL0 Support\n"); in has_32bit_el0()
1675 entry->desc); in has_useable_gicv3_cpuif()
1722 * Kdump isn't guaranteed to power-off all secondary CPUs, CNP in has_useable_cnp()
1774 * ThunderX leads to apparent I-cache corruption of kernel text, which in unmap_kernel_at_el0()
1783 __kpti_forced = -1; in unmap_kernel_at_el0()
1796 __kpti_forced = -1; in unmap_kernel_at_el0()
1870 #define KPTI_NG_TEMP_VA (-(1UL << PMD_SHIFT))
1881 kpti_ng_temp_alloc -= PAGE_SIZE; in kpti_ng_pgd_alloc()
1907 kpti_ng_temp_pgd = (pgd_t *)(alloc + (levels - 1) * PAGE_SIZE); in __kpti_install_ng_mappings()
1916 // +--------+-/-------+-/------ +-/------ +-\\\--------+ in __kpti_install_ng_mappings()
1918 // +--------+-\-------+-\------ +-\------ +-///--------+ in __kpti_install_ng_mappings()
1924 // to be used as a ad-hoc fixmap. in __kpti_install_ng_mappings()
1950 * We don't need to rewrite the page-tables if either we've done in kpti_install_ng_mappings()
1984 __kpti_forced = enabled ? 1 : -1; in parse_kpti()
2037 * DBM is a non-conflicting feature. i.e, the kernel can safely in has_hw_dbm()
2087 * The AMU extension is a non-conflicting feature: the kernel can in has_amu()
2094 * including secondary and hotplugged, marking this feature as in has_amu()
2120 * that, freshly-onlined CPUs will set tpidr_el2, so we don't need to in cpu_copy_el2regs()
2134 pr_warn("unavailable: %s\n", cap->desc); in has_nested_virt_support()
2177 * The ptr-auth feature levels are not intercompatible with lower in has_address_auth_cpucap()
2178 * levels. Hence we must match ptr-auth feature level of the secondary in has_address_auth_cpucap()
2179 * CPUs with that of the boot CPU. The level of boot cpu is fetched in has_address_auth_cpucap()
2181 * the secondary CPUs. in has_address_auth_cpucap()
2183 * boot CPU as a mismatched secondary CPU is parked before it gets in has_address_auth_cpucap()
2186 boot_val = cpuid_feature_extract_field(read_sanitised_ftr_reg(entry->sys_reg), in has_address_auth_cpucap()
2187 entry->field_pos, entry->sign); in has_address_auth_cpucap()
2189 return boot_val >= entry->min_field_value; in has_address_auth_cpucap()
2190 /* Now check for the secondary CPUs with SCOPE_LOCAL_CPU scope */ in has_address_auth_cpucap()
2191 sec_val = cpuid_feature_extract_field(__read_sysreg_by_encoding(entry->sys_reg), in has_address_auth_cpucap()
2192 entry->field_pos, entry->sign); in has_address_auth_cpucap()
2193 return (sec_val >= entry->min_field_value) && (sec_val == boot_val); in has_address_auth_cpucap()
2230 * ARM64_HAS_GIC_CPUIF_SYSREGS has a lower index, and is a boot CPU in can_use_gic_priorities()
2249 * ARM64_HAS_GIC_PRIO_MASKING has a lower index, and is a boot CPU in has_gic_prio_relaxed_sync()
2273 * Use of X16/X17 for tail-calls and trampolines that jump to in bti_enable()
2311 regp->user_mask &= ~ID_AA64ISAR1_EL1_BF16_MASK; in user_feature_fixup()
2319 regp->user_mask &= ~ID_AA64PFR1_EL1_SSBS_MASK; in user_feature_fixup()
2365 return !!(cap->type & ARM64_CPUCAP_OPTIONAL_FOR_LATE_CPU); in cpucap_late_cpu_optional()
2371 return !!(cap->type & ARM64_CPUCAP_PERMITTED_FOR_LATE_CPU); in cpucap_late_cpu_permitted()
2377 return !!(cap->type & ARM64_CPUCAP_PANIC_ON_CONFLICT); in cpucap_panic_on_conflict()
2462 .desc = "32-bit EL1 Support",
2563 .desc = "Stage-2 Force Write-Back",
2747 .desc = "RCpc load-acquire (LDAPR)",
2825 .desc = "Stage-1 Permission Indirection Extension (S1PIE)",
2845 .desc = "52-bit Virtual Addressing for KVM (LPA2)",
2864 .desc = "52-bit Virtual Addressing (LVA)",
2867 .desc = "52-bit Virtual Addressing (LPA2)",
2885 .desc = "Stage-1 Permission Overlay Extension (S1POE)",
2896 #define HWCAP_CPUID_MATCH(reg, field, min_value) \ argument
2898 ARM64_CPUID_FIELDS(reg, field, min_value)
2906 #define HWCAP_CAP(reg, field, min_value, cap_type, cap) \ argument
2909 HWCAP_CPUID_MATCH(reg, field, min_value) \
3069 * check is future proof, by making sure value is non-zero. in compat_has_neon()
3111 switch (cap->hwcap_type) { in cap_set_elf_hwcap()
3113 cpu_set_feature(cap->hwcap); in cap_set_elf_hwcap()
3117 compat_elf_hwcap |= (u32)cap->hwcap; in cap_set_elf_hwcap()
3120 compat_elf_hwcap2 |= (u32)cap->hwcap; in cap_set_elf_hwcap()
3134 switch (cap->hwcap_type) { in cpus_have_elf_hwcap()
3136 rc = cpu_have_feature(cap->hwcap); in cpus_have_elf_hwcap()
3140 rc = (compat_elf_hwcap & (u32)cap->hwcap) != 0; in cpus_have_elf_hwcap()
3143 rc = (compat_elf_hwcap2 & (u32)cap->hwcap) != 0; in cpus_have_elf_hwcap()
3158 for (; hwcaps->matches; hwcaps++) in setup_elf_hwcaps()
3159 if (hwcaps->matches(hwcaps, cpucap_default_scope(hwcaps))) in setup_elf_hwcaps()
3171 if (!caps || !(caps->type & scope_mask) || in update_cpu_capabilities()
3172 cpus_have_cap(caps->capability) || in update_cpu_capabilities()
3173 !caps->matches(caps, cpucap_default_scope(caps))) in update_cpu_capabilities()
3176 if (caps->desc && !caps->cpus) in update_cpu_capabilities()
3177 pr_info("detected: %s\n", caps->desc); in update_cpu_capabilities()
3179 __set_bit(caps->capability, system_cpucaps); in update_cpu_capabilities()
3181 if ((scope_mask & SCOPE_BOOT_CPU) && (caps->type & SCOPE_BOOT_CPU)) in update_cpu_capabilities()
3182 set_bit(caps->capability, boot_cpucaps); in update_cpu_capabilities()
3201 if (!(cap->type & non_boot_scope)) in cpu_enable_non_boot_scope_capabilities()
3204 if (cap->cpu_enable) in cpu_enable_non_boot_scope_capabilities()
3205 cap->cpu_enable(cap); in cpu_enable_non_boot_scope_capabilities()
3225 if (!caps || !(caps->type & scope_mask) || in enable_cpu_capabilities()
3226 !cpus_have_cap(caps->capability)) in enable_cpu_capabilities()
3229 if (boot_scope && caps->cpu_enable) in enable_cpu_capabilities()
3232 * before any secondary CPU boots. Thus, each secondary in enable_cpu_capabilities()
3235 * the boot CPU, for which the capability must be in enable_cpu_capabilities()
3239 caps->cpu_enable(caps); in enable_cpu_capabilities()
3243 * For all non-boot scope capabilities, use stop_machine() in enable_cpu_capabilities()
3268 if (!caps || !(caps->type & scope_mask)) in verify_local_cpu_caps()
3271 cpu_has_cap = caps->matches(caps, SCOPE_LOCAL_CPU); in verify_local_cpu_caps()
3272 system_has_cap = cpus_have_cap(caps->capability); in verify_local_cpu_caps()
3287 if (caps->cpu_enable) in verify_local_cpu_caps()
3288 caps->cpu_enable(caps); in verify_local_cpu_caps()
3301 smp_processor_id(), caps->capability, in verify_local_cpu_caps()
3302 caps->desc, system_has_cap, cpu_has_cap); in verify_local_cpu_caps()
3312 * Check for CPU features that are used in early boot
3313 * based on the Boot CPU value.
3326 for (; caps->matches; caps++) in __verify_local_elf_hwcaps()
3327 if (cpus_have_elf_hwcap(caps) && !caps->matches(caps, SCOPE_LOCAL_CPU)) { in __verify_local_elf_hwcaps()
3329 smp_processor_id(), caps->desc); in __verify_local_elf_hwcaps()
3401 * The capabilities were decided based on the available CPUs at the boot time.
3412 * on all secondary CPUs. in verify_local_cpu_capabilities()
3430 * All secondary CPUs should conform to the early CPU features in check_local_cpu_capabilities()
3431 * in use by the kernel based on boot CPU. in check_local_cpu_capabilities()
3453 return cap->matches(cap, SCOPE_LOCAL_CPU); in this_cpu_has_cap()
3462 * - The system wide safe registers are set with all the SMP CPUs and,
3463 * - The SYSTEM_FEATURE system_cpucaps may not have been set.
3471 return cap->matches(cap, SCOPE_SYSTEM); in __system_matches_cap()
3505 * The boot CPU's feature register values have been recorded. Detect in setup_boot_cpu_capabilities()
3506 * boot cpucaps and local cpucaps for the boot CPU, then enable and in setup_boot_cpu_capabilities()
3507 * patch alternatives for the available boot cpucaps. in setup_boot_cpu_capabilities()
3518 * handle the boot CPU. in setup_boot_cpu_features()
3523 * Detect broken pseudo-NMI. Must be called _before_ the call to in setup_boot_cpu_features()
3535 * The system-wide safe feature register values have been finalized. in setup_system_capabilities()
3550 if (caps && caps->cpus && caps->desc && in setup_system_capabilities()
3551 cpumask_any(caps->cpus) < nr_cpu_ids) in setup_system_capabilities()
3553 caps->desc, cpumask_pr_args(caps->cpus)); in setup_system_capabilities()
3597 * The first 32-bit-capable CPU we detected and so can no longer in enable_mismatched_32bit_el0()
3598 * be offlined by userspace. -1 indicates we haven't yet onlined in enable_mismatched_32bit_el0()
3599 * a 32-bit-capable CPU. in enable_mismatched_32bit_el0()
3601 static int lucky_winner = -1; in enable_mismatched_32bit_el0()
3604 bool cpu_32bit = id_aa64pfr0_32bit_el0(info->reg_id_aa64pfr0); in enable_mismatched_32bit_el0()
3619 * 32-bit EL0 online so that is_cpu_allowed() doesn't end up rejecting in enable_mismatched_32bit_el0()
3620 * every CPU in the system for a 32-bit task. in enable_mismatched_32bit_el0()
3624 get_cpu_device(lucky_winner)->offline_disabled = true; in enable_mismatched_32bit_el0()
3627 pr_info("Asymmetric 32-bit EL0 support detected on CPU %u; CPU hot-unplug disabled on CPU %u\n", in enable_mismatched_32bit_el0()
3638 return -ENOMEM; in init_32bit_el0_mask()
3653 * Op0 = 0x3, CRn = 0x0, Op1 = 0x0, CRm = [0, 2 - 7]
3654 * See Table C5-6 System instruction encodings for System register accesses,
3667 * With CRm == 0, reg should be one of :
3684 return -EINVAL; in emulate_id_reg()
3695 return -EINVAL; in emulate_sys_reg()