xref: /freebsd/lib/libc/amd64/string/amd64_archlevel.c (revision ad2fac552c3fcb577feb3055fa083f2d256c06ed)
1*ad2fac55SRobert Clausecker /*-
2*ad2fac55SRobert Clausecker  * Copyright (c) 2023 The FreeBSD Foundation
3*ad2fac55SRobert Clausecker  *
4*ad2fac55SRobert Clausecker  * This software was developed by Robert Clausecker <fuz@FreeBSD.org>
5*ad2fac55SRobert Clausecker  * under sponsorship from the FreeBSD Foundation.
6*ad2fac55SRobert Clausecker  *
7*ad2fac55SRobert Clausecker  * Redistribution and use in source and binary forms, with or without
8*ad2fac55SRobert Clausecker  * modification, are permitted provided that the following conditions
9*ad2fac55SRobert Clausecker  * are met:
10*ad2fac55SRobert Clausecker  * 1. Redistributions of source code must retain the above copyright
11*ad2fac55SRobert Clausecker  *    notice, this list of conditions and the following disclaimer.
12*ad2fac55SRobert Clausecker  * 2. Redistributions in binary form must reproduce the above copyright
13*ad2fac55SRobert Clausecker  *    notice, this list of conditions and the following disclaimer in the
14*ad2fac55SRobert Clausecker  *    documentation and/or other materials provided with the distribution.
15*ad2fac55SRobert Clausecker  *
16*ad2fac55SRobert Clausecker  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ''AS IS'' AND
17*ad2fac55SRobert Clausecker  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18*ad2fac55SRobert Clausecker  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19*ad2fac55SRobert Clausecker  * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20*ad2fac55SRobert Clausecker  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21*ad2fac55SRobert Clausecker  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22*ad2fac55SRobert Clausecker  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23*ad2fac55SRobert Clausecker  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24*ad2fac55SRobert Clausecker  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25*ad2fac55SRobert Clausecker  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26*ad2fac55SRobert Clausecker  * SUCH DAMAGE
27*ad2fac55SRobert Clausecker  */
28*ad2fac55SRobert Clausecker 
29*ad2fac55SRobert Clausecker #include <sys/types.h>
30*ad2fac55SRobert Clausecker 
31*ad2fac55SRobert Clausecker #include <machine/atomic.h>
32*ad2fac55SRobert Clausecker #include <machine/cpufunc.h>
33*ad2fac55SRobert Clausecker #include <machine/specialreg.h>
34*ad2fac55SRobert Clausecker 
35*ad2fac55SRobert Clausecker #include <stddef.h>
36*ad2fac55SRobert Clausecker #include <string.h>
37*ad2fac55SRobert Clausecker 
38*ad2fac55SRobert Clausecker #include "amd64_archlevel.h"
39*ad2fac55SRobert Clausecker #include "libc_private.h"
40*ad2fac55SRobert Clausecker 
41*ad2fac55SRobert Clausecker #define ARCHLEVEL_ENV	"ARCHLEVEL"
42*ad2fac55SRobert Clausecker 
43*ad2fac55SRobert Clausecker static volatile int amd64_archlevel = X86_64_UNDEFINED;
44*ad2fac55SRobert Clausecker 
45*ad2fac55SRobert Clausecker static const struct archlevel {
46*ad2fac55SRobert Clausecker 	char name[10];
47*ad2fac55SRobert Clausecker 	/* CPUID feature bits that need to be present */
48*ad2fac55SRobert Clausecker 	u_int feat_edx, feat_ecx, amd_ecx, ext_ebx;
49*ad2fac55SRobert Clausecker } levels[] = {
50*ad2fac55SRobert Clausecker 	{
51*ad2fac55SRobert Clausecker 		.name = "scalar",
52*ad2fac55SRobert Clausecker 		.feat_edx = 0,
53*ad2fac55SRobert Clausecker 		.feat_ecx = 0,
54*ad2fac55SRobert Clausecker 		.amd_ecx = 0,
55*ad2fac55SRobert Clausecker 		.ext_ebx = 0,
56*ad2fac55SRobert Clausecker 	}, {
57*ad2fac55SRobert Clausecker #define FEAT_EDX_BASELINE (CPUID_FPU | CPUID_CX8 | CPUID_CMOV | CPUID_MMX | \
58*ad2fac55SRobert Clausecker 	CPUID_FXSR | CPUID_SSE | CPUID_SSE2)
59*ad2fac55SRobert Clausecker 		.name = "baseline",
60*ad2fac55SRobert Clausecker 		.feat_edx = FEAT_EDX_BASELINE,
61*ad2fac55SRobert Clausecker 		.feat_ecx = 0,
62*ad2fac55SRobert Clausecker 		.amd_ecx = 0,
63*ad2fac55SRobert Clausecker 		.ext_ebx = 0,
64*ad2fac55SRobert Clausecker 	}, {
65*ad2fac55SRobert Clausecker #define FEAT_ECX_V2 (CPUID2_SSE3 | CPUID2_SSSE3 | CPUID2_CX16 | CPUID2_SSE41 | \
66*ad2fac55SRobert Clausecker 	CPUID2_SSE42 | CPUID2_POPCNT)
67*ad2fac55SRobert Clausecker #define AMD_ECX_V2 AMDID2_LAHF
68*ad2fac55SRobert Clausecker 		.name = "x86-64-v2",
69*ad2fac55SRobert Clausecker 		.feat_edx = FEAT_EDX_BASELINE,
70*ad2fac55SRobert Clausecker 		.feat_ecx = FEAT_ECX_V2,
71*ad2fac55SRobert Clausecker 		.amd_ecx = AMD_ECX_V2,
72*ad2fac55SRobert Clausecker 		.ext_ebx = 0,
73*ad2fac55SRobert Clausecker 	}, {
74*ad2fac55SRobert Clausecker #define FEAT_ECX_V3 (FEAT_ECX_V2 | CPUID2_FMA | CPUID2_MOVBE | \
75*ad2fac55SRobert Clausecker 	CPUID2_OSXSAVE | CPUID2_AVX | CPUID2_F16C)
76*ad2fac55SRobert Clausecker #define AMD_ECX_V3 (AMD_ECX_V2 | AMDID2_ABM)
77*ad2fac55SRobert Clausecker #define EXT_EBX_V3 (CPUID_STDEXT_BMI1 | CPUID_STDEXT_AVX2 | CPUID_STDEXT_BMI2)
78*ad2fac55SRobert Clausecker 		.name = "x86-64-v3",
79*ad2fac55SRobert Clausecker 		.feat_edx = FEAT_EDX_BASELINE,
80*ad2fac55SRobert Clausecker 		.feat_ecx = FEAT_ECX_V3,
81*ad2fac55SRobert Clausecker 		.amd_ecx = AMD_ECX_V3,
82*ad2fac55SRobert Clausecker 		.ext_ebx = EXT_EBX_V3,
83*ad2fac55SRobert Clausecker 	}, {
84*ad2fac55SRobert Clausecker #define EXT_EBX_V4 (EXT_EBX_V3 | CPUID_STDEXT_AVX512F | \
85*ad2fac55SRobert Clausecker 	CPUID_STDEXT_AVX512DQ | CPUID_STDEXT_AVX512CD | \
86*ad2fac55SRobert Clausecker 	CPUID_STDEXT_AVX512BW | CPUID_STDEXT_AVX512VL)
87*ad2fac55SRobert Clausecker 		.name = "x86-64-v4",
88*ad2fac55SRobert Clausecker 		.feat_edx = FEAT_EDX_BASELINE,
89*ad2fac55SRobert Clausecker 		.feat_ecx = FEAT_ECX_V3,
90*ad2fac55SRobert Clausecker 		.amd_ecx = AMD_ECX_V3,
91*ad2fac55SRobert Clausecker 		.ext_ebx = EXT_EBX_V4,
92*ad2fac55SRobert Clausecker 	}
93*ad2fac55SRobert Clausecker };
94*ad2fac55SRobert Clausecker 
95*ad2fac55SRobert Clausecker static int
supported_archlevel(u_int feat_edx,u_int feat_ecx,u_int ext_ebx,u_int ext_ecx)96*ad2fac55SRobert Clausecker supported_archlevel(u_int feat_edx, u_int feat_ecx, u_int ext_ebx, u_int ext_ecx)
97*ad2fac55SRobert Clausecker {
98*ad2fac55SRobert Clausecker 	int level;
99*ad2fac55SRobert Clausecker 	u_int p[4], max_leaf;
100*ad2fac55SRobert Clausecker 	u_int amd_ecx = 0;
101*ad2fac55SRobert Clausecker 
102*ad2fac55SRobert Clausecker 	(void)ext_ecx;
103*ad2fac55SRobert Clausecker 
104*ad2fac55SRobert Clausecker 	do_cpuid(0x80000000, p);
105*ad2fac55SRobert Clausecker 	max_leaf = p[0];
106*ad2fac55SRobert Clausecker 
107*ad2fac55SRobert Clausecker 	if (max_leaf >= 0x80000001) {
108*ad2fac55SRobert Clausecker 		do_cpuid(0x80000001, p);
109*ad2fac55SRobert Clausecker 		amd_ecx = p[2];
110*ad2fac55SRobert Clausecker 	}
111*ad2fac55SRobert Clausecker 
112*ad2fac55SRobert Clausecker 	for (level = X86_64_BASELINE; level <= X86_64_MAX; level++) {
113*ad2fac55SRobert Clausecker 		const struct archlevel *lvl = &levels[level];
114*ad2fac55SRobert Clausecker 
115*ad2fac55SRobert Clausecker 		if ((lvl->feat_edx & feat_edx) != lvl->feat_edx ||
116*ad2fac55SRobert Clausecker 		    (lvl->feat_ecx & feat_ecx) != lvl->feat_ecx ||
117*ad2fac55SRobert Clausecker 		    (lvl->amd_ecx & amd_ecx) != lvl->amd_ecx ||
118*ad2fac55SRobert Clausecker 		    (lvl->ext_ebx & ext_ebx) != lvl->ext_ebx)
119*ad2fac55SRobert Clausecker 			return (level - 1);
120*ad2fac55SRobert Clausecker 	}
121*ad2fac55SRobert Clausecker 
122*ad2fac55SRobert Clausecker 	return (X86_64_MAX);
123*ad2fac55SRobert Clausecker }
124*ad2fac55SRobert Clausecker 
125*ad2fac55SRobert Clausecker static int
match_archlevel(const char * str,int * force)126*ad2fac55SRobert Clausecker match_archlevel(const char *str, int *force)
127*ad2fac55SRobert Clausecker {
128*ad2fac55SRobert Clausecker 	int level, want_force = 0;
129*ad2fac55SRobert Clausecker 
130*ad2fac55SRobert Clausecker 	*force = 0;
131*ad2fac55SRobert Clausecker 
132*ad2fac55SRobert Clausecker 	if (str[0] == '!') {
133*ad2fac55SRobert Clausecker 		str++;
134*ad2fac55SRobert Clausecker 		want_force = 1;
135*ad2fac55SRobert Clausecker 	}
136*ad2fac55SRobert Clausecker 
137*ad2fac55SRobert Clausecker 	for (level = 0; level <= X86_64_MAX; level++) {
138*ad2fac55SRobert Clausecker 		size_t i;
139*ad2fac55SRobert Clausecker 		const char *candidate = levels[level].name;
140*ad2fac55SRobert Clausecker 
141*ad2fac55SRobert Clausecker 		/* can't use strcmp here: would recurse during ifunc resolution */
142*ad2fac55SRobert Clausecker 		for (i = 0; str[i] == candidate[i]; i++)
143*ad2fac55SRobert Clausecker 			/* suffixes starting with : or + are ignored for future extensions */
144*ad2fac55SRobert Clausecker 			if (str[i] == '\0' || str[i] == ':' || str[i] == '+') {
145*ad2fac55SRobert Clausecker 				if (want_force)
146*ad2fac55SRobert Clausecker 					*force = 1;
147*ad2fac55SRobert Clausecker 
148*ad2fac55SRobert Clausecker 				return (level);
149*ad2fac55SRobert Clausecker 			}
150*ad2fac55SRobert Clausecker 	}
151*ad2fac55SRobert Clausecker 
152*ad2fac55SRobert Clausecker 	return (X86_64_UNDEFINED);
153*ad2fac55SRobert Clausecker }
154*ad2fac55SRobert Clausecker 
155*ad2fac55SRobert Clausecker /*
156*ad2fac55SRobert Clausecker  * We can't use getenv(), strcmp(), and a bunch of other functions here as
157*ad2fac55SRobert Clausecker  * they may in turn call SIMD-optimised string functions.
158*ad2fac55SRobert Clausecker  *
159*ad2fac55SRobert Clausecker  * *force is set to 1 if the architecture level is valid and begins with a !
160*ad2fac55SRobert Clausecker  * and to 0 otherwise.
161*ad2fac55SRobert Clausecker  */
162*ad2fac55SRobert Clausecker static int
env_archlevel(int * force)163*ad2fac55SRobert Clausecker env_archlevel(int *force)
164*ad2fac55SRobert Clausecker {
165*ad2fac55SRobert Clausecker 	size_t i;
166*ad2fac55SRobert Clausecker 
167*ad2fac55SRobert Clausecker 	if (environ == NULL)
168*ad2fac55SRobert Clausecker 		return (X86_64_UNDEFINED);
169*ad2fac55SRobert Clausecker 
170*ad2fac55SRobert Clausecker 	for (i = 0; environ[i] != NULL; i++) {
171*ad2fac55SRobert Clausecker 		size_t j;
172*ad2fac55SRobert Clausecker 
173*ad2fac55SRobert Clausecker 		for (j = 0; environ[i][j] == ARCHLEVEL_ENV "="[j]; j++)
174*ad2fac55SRobert Clausecker 			if (environ[i][j] == '=')
175*ad2fac55SRobert Clausecker 				return (match_archlevel(&environ[i][j + 1], force));
176*ad2fac55SRobert Clausecker 	}
177*ad2fac55SRobert Clausecker 
178*ad2fac55SRobert Clausecker 	*force = 0;
179*ad2fac55SRobert Clausecker 
180*ad2fac55SRobert Clausecker 	return (X86_64_UNDEFINED);
181*ad2fac55SRobert Clausecker 
182*ad2fac55SRobert Clausecker }
183*ad2fac55SRobert Clausecker 
184*ad2fac55SRobert Clausecker /*
185*ad2fac55SRobert Clausecker  * Determine the architecture level by checking the CPU capabilities
186*ad2fac55SRobert Clausecker  * and the environment:
187*ad2fac55SRobert Clausecker  *
188*ad2fac55SRobert Clausecker  * 1. If environment variable ARCHLEVEL starts with a ! and is followed
189*ad2fac55SRobert Clausecker  *    by a valid architecture level, that level is returned.
190*ad2fac55SRobert Clausecker  * 2. Else if ARCHLEVEL is set to a valid architecture level that is
191*ad2fac55SRobert Clausecker  *    supported by the CPU, that level is returned.
192*ad2fac55SRobert Clausecker  * 3. Else the highest architecture level supported by the CPU is
193*ad2fac55SRobert Clausecker  *    returned.
194*ad2fac55SRobert Clausecker  *
195*ad2fac55SRobert Clausecker  * Valid architecture levels are those defined in the levels array.
196*ad2fac55SRobert Clausecker  * The architecture level "scalar" indicates that SIMD enhancements
197*ad2fac55SRobert Clausecker  * shall not be used.
198*ad2fac55SRobert Clausecker  */
199*ad2fac55SRobert Clausecker static int
archlevel(u_int feat_edx,u_int feat_ecx,u_int ext_ebx,u_int ext_ecx)200*ad2fac55SRobert Clausecker archlevel(u_int feat_edx, u_int feat_ecx, u_int ext_ebx, u_int ext_ecx)
201*ad2fac55SRobert Clausecker {
202*ad2fac55SRobert Clausecker 	int islevel, wantlevel, hwlevel, force;
203*ad2fac55SRobert Clausecker 
204*ad2fac55SRobert Clausecker 	islevel = atomic_load_int(&amd64_archlevel);
205*ad2fac55SRobert Clausecker 	if (islevel != X86_64_UNDEFINED)
206*ad2fac55SRobert Clausecker 		return (islevel);
207*ad2fac55SRobert Clausecker 
208*ad2fac55SRobert Clausecker 	wantlevel = env_archlevel(&force);
209*ad2fac55SRobert Clausecker 	if (!force) {
210*ad2fac55SRobert Clausecker 		hwlevel = supported_archlevel(feat_edx, feat_ecx, ext_ebx, ext_ecx);
211*ad2fac55SRobert Clausecker 		if (wantlevel == X86_64_UNDEFINED || wantlevel > hwlevel)
212*ad2fac55SRobert Clausecker 			wantlevel = hwlevel;
213*ad2fac55SRobert Clausecker 	}
214*ad2fac55SRobert Clausecker 
215*ad2fac55SRobert Clausecker 	/*
216*ad2fac55SRobert Clausecker 	 * Ensure amd64_archlevel is set only once and
217*ad2fac55SRobert Clausecker 	 * all calls agree on what it was set to.
218*ad2fac55SRobert Clausecker 	 */
219*ad2fac55SRobert Clausecker 	if (atomic_cmpset_int(&amd64_archlevel, islevel, wantlevel))
220*ad2fac55SRobert Clausecker 		return (wantlevel);
221*ad2fac55SRobert Clausecker 	else
222*ad2fac55SRobert Clausecker 		return (atomic_load_int(&amd64_archlevel));
223*ad2fac55SRobert Clausecker }
224*ad2fac55SRobert Clausecker 
225*ad2fac55SRobert Clausecker /*
226*ad2fac55SRobert Clausecker  * Helper function for SIMD ifunc dispatch: select the highest level
227*ad2fac55SRobert Clausecker  * implementation up to the current architecture level.
228*ad2fac55SRobert Clausecker  */
229*ad2fac55SRobert Clausecker dlfunc_t
__archlevel_resolve(u_int feat_edx,u_int feat_ecx,u_int ext_ebx,u_int ext_ecx,int32_t funcs[static X86_64_MAX+1])230*ad2fac55SRobert Clausecker __archlevel_resolve(u_int feat_edx, u_int feat_ecx, u_int ext_ebx,
231*ad2fac55SRobert Clausecker     u_int ext_ecx, int32_t funcs[static X86_64_MAX + 1])
232*ad2fac55SRobert Clausecker {
233*ad2fac55SRobert Clausecker 	int level;
234*ad2fac55SRobert Clausecker 
235*ad2fac55SRobert Clausecker 	for (level = archlevel(feat_edx, feat_ecx, ext_ebx, ext_ecx); level >= 0; level--)
236*ad2fac55SRobert Clausecker 		if (funcs[level] != 0)
237*ad2fac55SRobert Clausecker 			return (dlfunc_t)((uintptr_t)funcs + (ptrdiff_t)funcs[level]);
238*ad2fac55SRobert Clausecker 
239*ad2fac55SRobert Clausecker 	/* no function is present -- what now? */
240*ad2fac55SRobert Clausecker 	__builtin_trap();
241*ad2fac55SRobert Clausecker }
242