1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * CDDL HEADER START 4 * 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or https://opensource.org/licenses/CDDL-1.0. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2024 Google, Inc. All rights reserved. 24 */ 25 #include <sys/zfs_context.h> 26 #include <sys/kstat.h> 27 #include <sys/simd.h> 28 29 30 #ifdef _KERNEL 31 #ifdef __linux__ 32 #include <linux/simd.h> 33 #endif /* __linux__ */ 34 kstat_t *simd_stat_kstat; 35 #endif /* _KERNEL */ 36 37 #ifdef _KERNEL 38 /* Sometimes, we don't define these at all. */ 39 #ifndef HAVE_KERNEL_FPU 40 #define HAVE_KERNEL_FPU (0) 41 #endif 42 #ifndef HAVE_KERNEL_NEON 43 #define HAVE_KERNEL_NEON (0) 44 #endif 45 #ifndef HAVE_KERNEL_FPU_INTERNAL 46 #define HAVE_KERNEL_FPU_INTERNAL (0) 47 #endif 48 #ifndef HAVE_UNDERSCORE_KERNEL_FPU 49 #define HAVE_UNDERSCORE_KERNEL_FPU (0) 50 #endif 51 52 #define SIMD_STAT_PRINT(s, feat, val) \ 53 kmem_scnprintf(s + off, MAX(4095-off, 0), "%-16s\t%1d\n", feat, (val)) 54 55 static int 56 simd_stat_kstat_data(char *buf, size_t size, void *data) 57 { 58 (void) data; 59 60 static char simd_stat_kstat_payload[4096] = {0}; 61 static int off = 0; 62 #ifdef __linux__ 63 if (off == 0) { 64 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 65 "kfpu_allowed", kfpu_allowed()); 66 #if defined(__x86_64__) || defined(__i386__) 67 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 68 "kfpu", HAVE_KERNEL_FPU); 69 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 70 "kfpu_internal", HAVE_KERNEL_FPU_INTERNAL); 71 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 72 "__kernel_fpu", HAVE_UNDERSCORE_KERNEL_FPU); 73 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 74 "sse", zfs_sse_available()); 75 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 76 "sse2", zfs_sse2_available()); 77 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 78 "sse3", zfs_sse3_available()); 79 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 80 "ssse3", zfs_ssse3_available()); 81 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 82 "sse41", zfs_sse4_1_available()); 83 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 84 "sse42", zfs_sse4_2_available()); 85 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 86 "avx", zfs_avx_available()); 87 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 88 "avx2", zfs_avx2_available()); 89 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 90 "avx512f", zfs_avx512f_available()); 91 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 92 "avx512cd", zfs_avx512cd_available()); 93 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 94 "avx512er", zfs_avx512er_available()); 95 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 96 "avx512pf", zfs_avx512pf_available()); 97 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 98 "avx512bw", zfs_avx512bw_available()); 99 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 100 "avx512dq", zfs_avx512dq_available()); 101 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 102 "avx512vl", zfs_avx512vl_available()); 103 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 104 "avx512ifma", zfs_avx512ifma_available()); 105 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 106 "avx512vbmi", zfs_avx512vbmi_available()); 107 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 108 "ymm", __ymm_enabled()); 109 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 110 "zmm", __zmm_enabled()); 111 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 112 "bmi1", zfs_bmi1_available()); 113 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 114 "bmi2", zfs_bmi2_available()); 115 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 116 "aes", zfs_aes_available()); 117 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 118 "pclmulqdq", zfs_pclmulqdq_available()); 119 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 120 "movbe", zfs_movbe_available()); 121 122 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 123 "osxsave", boot_cpu_has(X86_FEATURE_OSXSAVE)); 124 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 125 "xsaves", static_cpu_has(X86_FEATURE_XSAVES)); 126 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 127 "xsaveopt", static_cpu_has(X86_FEATURE_XSAVEOPT)); 128 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 129 "xsave", static_cpu_has(X86_FEATURE_XSAVE)); 130 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 131 "fxsr", static_cpu_has(X86_FEATURE_FXSR)); 132 #endif /* __x86__ */ 133 #if defined(__arm__) || defined(__aarch64__) 134 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 135 "kernel_neon", HAVE_KERNEL_NEON); 136 #if defined(CONFIG_KERNEL_MODE_NEON) 137 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 138 "kernel_mode_neon", CONFIG_KERNEL_MODE_NEON); 139 #endif /* CONFIG_KERNEL_MODE_NEON */ 140 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 141 "neon", zfs_neon_available()); 142 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 143 "sha256", zfs_sha256_available()); 144 #if defined(__aarch64__) 145 /* 146 * This technically can exist on 32b ARM but we don't 147 * define hooks to check for it and I didn't want to 148 * learn enough ARM ASM to add one. 149 */ 150 off += SIMD_STAT_PRINT(simd_stat_kstat_payload, 151 "sha512", zfs_sha512_available()); 152 #endif /* __aarch64__ */ 153 #endif /* __arm__ */ 154 /* We want to short-circuit this on unsupported platforms. */ 155 off += 1; 156 } 157 158 kmem_scnprintf(buf, MIN(off, size), "%s", simd_stat_kstat_payload); 159 #endif /* __linux__ */ 160 return (0); 161 } 162 #endif /* _KERNEL */ 163 164 void 165 simd_stat_init(void) 166 { 167 static boolean_t simd_stat_initialized = B_FALSE; 168 169 if (!simd_stat_initialized) { 170 #if defined(_KERNEL) 171 /* Install kstats for all implementations */ 172 simd_stat_kstat = kstat_create("zfs", 0, "simd", "misc", 173 KSTAT_TYPE_RAW, 0, KSTAT_FLAG_VIRTUAL); 174 175 176 if (simd_stat_kstat != NULL) { 177 simd_stat_kstat->ks_data = (void*)(uintptr_t)1; 178 simd_stat_kstat->ks_ndata = 1; 179 simd_stat_kstat->ks_flags |= KSTAT_FLAG_NO_HEADERS; 180 kstat_set_raw_ops(simd_stat_kstat, 181 NULL, 182 simd_stat_kstat_data, 183 NULL); 184 kstat_install(simd_stat_kstat); 185 } 186 #endif /* _KERNEL */ 187 } 188 /* Finish initialization */ 189 simd_stat_initialized = B_TRUE; 190 } 191 192 void 193 simd_stat_fini(void) 194 { 195 #if defined(_KERNEL) 196 if (simd_stat_kstat != NULL) { 197 kstat_delete(simd_stat_kstat); 198 simd_stat_kstat = NULL; 199 } 200 #endif 201 } 202 203 #ifdef _KERNEL 204 EXPORT_SYMBOL(simd_stat_init); 205 EXPORT_SYMBOL(simd_stat_fini); 206 #endif 207