1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * This file and its contents are supplied under the terms of the 4 * Common Development and Distribution License ("CDDL"), version 1.0. 5 * You may only use this file in accordance with the terms of version 6 * 1.0 of the CDDL. 7 * 8 * A full copy of the text of the CDDL should have accompanied this 9 * source. A copy of the CDDL is also available via the Internet at 10 * https://opensource.org/license/CDDL-1.0. 11 */ 12 /* 13 * Copyright 2024 Google, Inc. All rights reserved. 14 */ 15 #include <sys/zfs_context.h> 16 #include <sys/kstat.h> 17 #include <sys/simd.h> 18 19 20 #ifdef _KERNEL 21 #ifdef __linux__ 22 #include <linux/simd.h> 23 #endif /* __linux__ */ 24 kstat_t *simd_stat_kstat; 25 #endif /* _KERNEL */ 26 27 #ifdef _KERNEL 28 /* Sometimes, we don't define these at all. */ 29 #ifndef HAVE_KERNEL_FPU 30 #define HAVE_KERNEL_FPU (0) 31 #endif 32 #ifndef HAVE_KERNEL_NEON 33 #define HAVE_KERNEL_NEON (0) 34 #endif 35 #ifndef HAVE_KERNEL_FPU_INTERNAL 36 #define HAVE_KERNEL_FPU_INTERNAL (0) 37 #endif 38 #ifndef HAVE_UNDERSCORE_KERNEL_FPU 39 #define HAVE_UNDERSCORE_KERNEL_FPU (0) 40 #endif 41 42 #define SIMD_STAT_PRINT(s, feat, val) \ 43 kmem_scnprintf(s + off, MAX(4095-off, 0), "%-16s\t%1d\n", feat, (val)) 44 45 static int 46 simd_stat_kstat_data(char *buf, size_t size, void *data) 47 { 48 (void) data; 49 50 static char simd_stat_kstat_payload[4096] = {0}; 51 static int off = 0; 52 #ifdef __linux__ 53 if (off == 0) { 54 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 55 "kfpu_allowed", kfpu_allowed()); 56 #if defined(__x86_64__) || defined(__i386__) 57 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 58 "kfpu", HAVE_KERNEL_FPU); 59 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 60 "kfpu_internal", HAVE_KERNEL_FPU_INTERNAL); 61 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 62 "__kernel_fpu", HAVE_UNDERSCORE_KERNEL_FPU); 63 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 64 "sse", zfs_sse_available()); 65 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 66 "sse2", zfs_sse2_available()); 67 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 68 "sse3", zfs_sse3_available()); 69 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 70 "ssse3", zfs_ssse3_available()); 71 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 72 "sse41", zfs_sse4_1_available()); 73 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 74 "sse42", zfs_sse4_2_available()); 75 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 76 "avx", zfs_avx_available()); 77 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 78 "avx2", zfs_avx2_available()); 79 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 80 "avx512f", zfs_avx512f_available()); 81 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 82 "avx512cd", zfs_avx512cd_available()); 83 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 84 "avx512er", zfs_avx512er_available()); 85 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 86 "avx512pf", zfs_avx512pf_available()); 87 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 88 "avx512bw", zfs_avx512bw_available()); 89 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 90 "avx512dq", zfs_avx512dq_available()); 91 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 92 "avx512vl", zfs_avx512vl_available()); 93 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 94 "avx512ifma", zfs_avx512ifma_available()); 95 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 96 "avx512vbmi", zfs_avx512vbmi_available()); 97 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 98 "ymm", __ymm_enabled()); 99 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 100 "zmm", __zmm_enabled()); 101 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 102 "bmi1", zfs_bmi1_available()); 103 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 104 "bmi2", zfs_bmi2_available()); 105 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 106 "aes", zfs_aes_available()); 107 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 108 "pclmulqdq", zfs_pclmulqdq_available()); 109 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 110 "movbe", zfs_movbe_available()); 111 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 112 "vaes", zfs_vaes_available()); 113 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 114 "vpclmulqdq", zfs_vpclmulqdq_available()); 115 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 116 "sha512ext", zfs_sha512ext_available()); 117 118 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 119 "osxsave", boot_cpu_has(X86_FEATURE_OSXSAVE)); 120 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 121 "xsaves", cpu_feature_enabled(X86_FEATURE_XSAVES)); 122 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 123 "xsaveopt", cpu_feature_enabled(X86_FEATURE_XSAVEOPT)); 124 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 125 "xsave", cpu_feature_enabled(X86_FEATURE_XSAVE)); 126 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 127 "fxsr", cpu_feature_enabled(X86_FEATURE_FXSR)); 128 #endif /* __x86__ */ 129 #if defined(__arm__) || defined(__aarch64__) 130 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 131 "kernel_neon", HAVE_KERNEL_NEON); 132 #if defined(CONFIG_KERNEL_MODE_NEON) 133 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 134 "kernel_mode_neon", CONFIG_KERNEL_MODE_NEON); 135 #endif /* CONFIG_KERNEL_MODE_NEON */ 136 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 137 "neon", zfs_neon_available()); 138 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 139 "sha256", zfs_sha256_available()); 140 #if defined(__aarch64__) 141 /* 142 * This technically can exist on 32b ARM but we don't 143 * define hooks to check for it and I didn't want to 144 * learn enough ARM ASM to add one. 145 */ 146 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 147 "sha512", zfs_sha512_available()); 148 #endif /* __aarch64__ */ 149 #endif /* __arm__ */ 150 /* We want to short-circuit this on unsupported platforms. */ 151 off += 1; 152 } 153 154 kmem_scnprintf(buf, MIN(off, size), "%s", simd_stat_kstat_payload); 155 #endif /* __linux__ */ 156 return (0); 157 } 158 #endif /* _KERNEL */ 159 160 void 161 simd_stat_init(void) 162 { 163 static boolean_t simd_stat_initialized = B_FALSE; 164 165 if (!simd_stat_initialized) { 166 #if defined(_KERNEL) 167 /* Install kstats for all implementations */ 168 simd_stat_kstat = kstat_create("zfs", 0, "simd", "misc", 169 KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VIRTUAL); 170 171 172 if (simd_stat_kstat != NULL) { 173 simd_stat_kstat->ks_data = (void*)(uintptr_t)1; 174 simd_stat_kstat->ks_ndata = 1; 175 simd_stat_kstat->ks_flags |= KSTAT_FLAG_NO_HEADERS; 176 kstat_set_raw_ops(simd_stat_kstat, 177 NULL, 178 simd_stat_kstat_data, 179 NULL); 180 kstat_install(simd_stat_kstat); 181 } 182 #endif /* _KERNEL */ 183 } 184 /* Finish initialization */ 185 simd_stat_initialized = B_TRUE; 186 } 187 188 void 189 simd_stat_fini(void) 190 { 191 #if defined(_KERNEL) 192 if (simd_stat_kstat != NULL) { 193 kstat_delete(simd_stat_kstat); 194 simd_stat_kstat = NULL; 195 } 196 #endif 197 } 198 199 #ifdef _KERNEL 200 EXPORT_SYMBOL(simd_stat_init); 201 EXPORT_SYMBOL(simd_stat_fini); 202 #endif 203