1#if __has_include(<sys/auxv.h>) 2#include <sys/auxv.h> 3#define HAVE_SYS_AUXV_H 4#endif 5 6static void __init_cpu_features_constructor(unsigned long hwcap, 7 const __ifunc_arg_t *arg) { 8 unsigned long long feat = 0; 9#define setCPUFeature(F) feat |= 1ULL << F 10#define getCPUFeature(id, ftr) __asm__("mrs %0, " #id : "=r"(ftr)) 11#define extractBits(val, start, number) \ 12 (val & ((1ULL << number) - 1ULL) << start) >> start 13 unsigned long hwcap2 = 0; 14 if (hwcap & _IFUNC_ARG_HWCAP) 15 hwcap2 = arg->_hwcap2; 16 if (hwcap & HWCAP_CRC32) 17 setCPUFeature(FEAT_CRC); 18 if (hwcap & HWCAP_PMULL) 19 setCPUFeature(FEAT_PMULL); 20 if (hwcap & HWCAP_FLAGM) 21 setCPUFeature(FEAT_FLAGM); 22 if (hwcap2 & HWCAP2_FLAGM2) 23 setCPUFeature(FEAT_FLAGM2); 24 if (hwcap & HWCAP_SM4) 25 setCPUFeature(FEAT_SM4); 26 if (hwcap & HWCAP_ASIMDDP) 27 setCPUFeature(FEAT_DOTPROD); 28 if (hwcap & HWCAP_ASIMDFHM) 29 setCPUFeature(FEAT_FP16FML); 30 if (hwcap & HWCAP_FPHP) 31 setCPUFeature(FEAT_FP16); 32 if (hwcap & HWCAP_DIT) 33 setCPUFeature(FEAT_DIT); 34 if (hwcap & HWCAP_ASIMDRDM) 35 setCPUFeature(FEAT_RDM); 36 if (hwcap & HWCAP_AES) 37 setCPUFeature(FEAT_AES); 38 if (hwcap & HWCAP_SHA1) 39 setCPUFeature(FEAT_SHA1); 40 if (hwcap & HWCAP_SHA2) 41 setCPUFeature(FEAT_SHA2); 42 if (hwcap & HWCAP_JSCVT) 43 setCPUFeature(FEAT_JSCVT); 44 if (hwcap & HWCAP_FCMA) 45 setCPUFeature(FEAT_FCMA); 46 if (hwcap & HWCAP_SB) 47 setCPUFeature(FEAT_SB); 48 if (hwcap & HWCAP_SSBS) { 49 setCPUFeature(FEAT_SSBS); 50 setCPUFeature(FEAT_SSBS2); 51 } 52 if (hwcap2 & HWCAP2_MTE) { 53 setCPUFeature(FEAT_MEMTAG); 54 setCPUFeature(FEAT_MEMTAG2); 55 } 56 if (hwcap2 & HWCAP2_MTE3) 57 setCPUFeature(FEAT_MEMTAG3); 58 if (hwcap2 & HWCAP2_SVEAES) 59 setCPUFeature(FEAT_SVE_AES); 60 if (hwcap2 & HWCAP2_SVEPMULL) 61 setCPUFeature(FEAT_SVE_PMULL128); 62 if (hwcap2 & HWCAP2_SVEBITPERM) 63 setCPUFeature(FEAT_SVE_BITPERM); 64 if (hwcap2 & HWCAP2_SVESHA3) 65 setCPUFeature(FEAT_SVE_SHA3); 66 if (hwcap2 & HWCAP2_SVESM4) 67 setCPUFeature(FEAT_SVE_SM4); 68 if (hwcap2 & HWCAP2_DCPODP) 69 setCPUFeature(FEAT_DPB2); 70 if (hwcap & HWCAP_ATOMICS) 71 setCPUFeature(FEAT_LSE); 72 if (hwcap2 & HWCAP2_RNG) 73 setCPUFeature(FEAT_RNG); 74 if (hwcap2 & HWCAP2_I8MM) 75 setCPUFeature(FEAT_I8MM); 76 if (hwcap2 & HWCAP2_EBF16) 77 setCPUFeature(FEAT_EBF16); 78 if (hwcap2 & HWCAP2_SVE_EBF16) 79 setCPUFeature(FEAT_SVE_EBF16); 80 if (hwcap2 & HWCAP2_DGH) 81 setCPUFeature(FEAT_DGH); 82 if (hwcap2 & HWCAP2_FRINT) 83 setCPUFeature(FEAT_FRINTTS); 84 if (hwcap2 & HWCAP2_SVEI8MM) 85 setCPUFeature(FEAT_SVE_I8MM); 86 if (hwcap2 & HWCAP2_SVEF32MM) 87 setCPUFeature(FEAT_SVE_F32MM); 88 if (hwcap2 & HWCAP2_SVEF64MM) 89 setCPUFeature(FEAT_SVE_F64MM); 90 if (hwcap2 & HWCAP2_BTI) 91 setCPUFeature(FEAT_BTI); 92 if (hwcap2 & HWCAP2_RPRES) 93 setCPUFeature(FEAT_RPRES); 94 if (hwcap2 & HWCAP2_WFXT) 95 setCPUFeature(FEAT_WFXT); 96 if (hwcap2 & HWCAP2_SME) 97 setCPUFeature(FEAT_SME); 98 if (hwcap2 & HWCAP2_SME2) 99 setCPUFeature(FEAT_SME2); 100 if (hwcap2 & HWCAP2_SME_I16I64) 101 setCPUFeature(FEAT_SME_I64); 102 if (hwcap2 & HWCAP2_SME_F64F64) 103 setCPUFeature(FEAT_SME_F64); 104 if (hwcap2 & HWCAP2_MOPS) 105 setCPUFeature(FEAT_MOPS); 106 if (hwcap & HWCAP_CPUID) { 107 unsigned long ftr; 108 109 getCPUFeature(ID_AA64ISAR1_EL1, ftr); 110 /* ID_AA64ISAR1_EL1.SPECRES >= 0b0001 */ 111 if (extractBits(ftr, 40, 4) >= 0x1) 112 setCPUFeature(FEAT_PREDRES); 113 /* ID_AA64ISAR1_EL1.LS64 >= 0b0001 */ 114 if (extractBits(ftr, 60, 4) >= 0x1) 115 setCPUFeature(FEAT_LS64); 116 /* ID_AA64ISAR1_EL1.LS64 >= 0b0010 */ 117 if (extractBits(ftr, 60, 4) >= 0x2) 118 setCPUFeature(FEAT_LS64_V); 119 /* ID_AA64ISAR1_EL1.LS64 >= 0b0011 */ 120 if (extractBits(ftr, 60, 4) >= 0x3) 121 setCPUFeature(FEAT_LS64_ACCDATA); 122 } 123 if (hwcap & HWCAP_FP) { 124 setCPUFeature(FEAT_FP); 125 // FP and AdvSIMD fields have the same value 126 setCPUFeature(FEAT_SIMD); 127 } 128 if (hwcap & HWCAP_DCPOP) 129 setCPUFeature(FEAT_DPB); 130 if (hwcap & HWCAP_LRCPC) 131 setCPUFeature(FEAT_RCPC); 132 if (hwcap & HWCAP_ILRCPC) 133 setCPUFeature(FEAT_RCPC2); 134 if (hwcap2 & HWCAP2_LRCPC3) 135 setCPUFeature(FEAT_RCPC3); 136 if (hwcap2 & HWCAP2_BF16) 137 setCPUFeature(FEAT_BF16); 138 if (hwcap2 & HWCAP2_SVEBF16) 139 setCPUFeature(FEAT_SVE_BF16); 140 if (hwcap & HWCAP_SVE) 141 setCPUFeature(FEAT_SVE); 142 if (hwcap2 & HWCAP2_SVE2) 143 setCPUFeature(FEAT_SVE2); 144 if (hwcap & HWCAP_SHA3) 145 setCPUFeature(FEAT_SHA3); 146 setCPUFeature(FEAT_INIT); 147 148 __atomic_store_n(&__aarch64_cpu_features.features, feat, __ATOMIC_RELAXED); 149} 150