1 /* x86 CPU feature tuning.
2 This file is part of the GNU C Library.
3 Copyright (C) 2017-2023 Free Software Foundation, Inc.
5 The GNU C Library is free software; you can redistribute it and/or
6 modify it under the terms of the GNU Lesser General Public
7 License as published by the Free Software Foundation; either
8 version 2.1 of the License, or (at your option) any later version.
10 The GNU C Library is distributed in the hope that it will be useful,
11 but WITHOUT ANY WARRANTY; without even the implied warranty of
12 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 Lesser General Public License for more details.
15 You should have received a copy of the GNU Lesser General Public
16 License along with the GNU C Library; if not, see
17 <https://www.gnu.org/licenses/>. */
19 #define TUNABLE_NAMESPACE cpu
22 #include <unistd.h> /* Get STDOUT_FILENO for _dl_printf. */
23 #include <elf/dl-tunables.h>
25 #include <cpu-features.h>
27 #include <dl-tunables-parse.h>
28 #include <dl-symbol-redir-ifunc.h>
30 #define CHECK_GLIBC_IFUNC_CPU_OFF(f, cpu_features, name, len) \
31 _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \
32 if (tunable_str_comma_strcmp_cte (&f, #name)) \
34 CPU_FEATURE_UNSET (cpu_features, name) \
38 /* Disable a preferred feature NAME. We don't enable a preferred feature
39 which isn't available. */
40 #define CHECK_GLIBC_IFUNC_PREFERRED_OFF(f, cpu_features, name, len) \
41 _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \
42 if (tunable_str_comma_strcmp_cte (&f, #name) == 0) \
44 cpu_features->preferred[index_arch_##name] \
45 &= ~bit_arch_##name; \
49 /* Enable/disable a preferred feature NAME. */
50 #define CHECK_GLIBC_IFUNC_PREFERRED_BOTH(f, cpu_features, name, len) \
51 _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \
52 if (tunable_str_comma_strcmp_cte (&f, #name)) \
55 cpu_features->preferred[index_arch_##name] &= ~bit_arch_##name; \
57 cpu_features->preferred[index_arch_##name] |= bit_arch_##name; \
61 /* Enable/disable a preferred feature NAME. Enable a preferred feature
62 only if the feature NEED is usable. */
63 #define CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH(f, cpu_features, name, \
65 _Static_assert (sizeof (#name) - 1 == len, #name " != " #len); \
66 if (tunable_str_comma_strcmp_cte (&f, #name)) \
69 cpu_features->preferred[index_arch_##name] &= ~bit_arch_##name; \
70 else if (CPU_FEATURE_USABLE_P (cpu_features, need)) \
71 cpu_features->preferred[index_arch_##name] |= bit_arch_##name; \
77 TUNABLE_CALLBACK (set_hwcaps
) (tunable_val_t
*valp
)
79 /* The current IFUNC selection is based on microbenchmarks in glibc.
80 It should give the best performance for most workloads. But other
81 choices may have better performance for a particular workload or on
82 the hardware which wasn't available when the selection was made.
83 The environment variable:
85 GLIBC_TUNABLES=glibc.cpu.hwcaps=-xxx,yyy,-zzz,....
87 can be used to enable CPU/ARCH feature yyy, disable CPU/ARCH feature
88 yyy and zzz, where the feature name is case-sensitive and has to
89 match the ones in cpu-features.h. It can be used by glibc developers
90 to tune for a new processor or override the IFUNC selection to
91 improve performance for a particular workload.
93 NOTE: the IFUNC selection may change over time. Please check all
94 multiarch implementations when experimenting. */
96 struct cpu_features
*cpu_features
= &GLRO(dl_x86_cpu_features
);
98 struct tunable_str_comma_state_t ts
;
99 tunable_str_comma_init (&ts
, valp
);
101 struct tunable_str_comma_t n
;
102 while (tunable_str_comma_next (&ts
, &n
))
111 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, AVX
, 3);
112 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, CX8
, 3);
113 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, FMA
, 3);
114 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, HTT
, 3);
115 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, IBT
, 3);
116 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, RTM
, 3);
122 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, AVX2
, 4);
123 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, BMI1
, 4);
124 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, BMI2
, 4);
125 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, CMOV
, 4);
126 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, ERMS
, 4);
127 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, FMA4
, 4);
128 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, SSE2
, 4);
129 CHECK_GLIBC_IFUNC_PREFERRED_OFF (n
, cpu_features
, I586
, 4);
130 CHECK_GLIBC_IFUNC_PREFERRED_OFF (n
, cpu_features
, I686
, 4);
136 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, LZCNT
, 5);
137 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, MOVBE
, 5);
138 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, SHSTK
, 5);
139 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, SSSE3
, 5);
140 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, XSAVE
, 5);
146 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, POPCNT
, 6);
147 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, SSE4_1
, 6);
148 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, SSE4_2
, 6);
149 if (memcmp (n
.str
, "XSAVEC", 6) == 0)
151 /* Update xsave_state_size to XSAVE state size. */
152 cpu_features
->xsave_state_size
153 = cpu_features
->xsave_state_full_size
;
154 CPU_FEATURE_UNSET (cpu_features
, XSAVEC
);
161 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, AVX512F
, 7);
162 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, OSXSAVE
, 7);
168 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, AVX512CD
, 8);
169 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, AVX512BW
, 8);
170 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, AVX512DQ
, 8);
171 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, AVX512ER
, 8);
172 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, AVX512PF
, 8);
173 CHECK_GLIBC_IFUNC_CPU_OFF (n
, cpu_features
, AVX512VL
, 8);
175 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n
, cpu_features
, Slow_BSF
, 8);
179 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n
, cpu_features
, Prefer_ERMS
,
181 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n
, cpu_features
, Prefer_FSRM
,
183 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH (n
, cpu_features
,
191 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n
, cpu_features
,
192 Fast_Rep_String
, 15);
197 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
198 (n
, cpu_features
, Prefer_No_AVX512
, AVX512F
, 16);
203 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n
, cpu_features
,
204 Fast_Copy_Backward
, 18);
209 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n
, cpu_features
,
210 Fast_Unaligned_Load
, 19);
211 CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n
, cpu_features
,
212 Fast_Unaligned_Copy
, 19);
217 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
218 (n
, cpu_features
, Prefer_No_VZEROUPPER
, AVX
, 20);
223 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
224 (n
, cpu_features
, AVX_Fast_Unaligned_Load
, AVX
, 23);
229 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
230 (n
, cpu_features
, MathVec_Prefer_No_AVX512
, AVX512F
, 24);
235 CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
236 (n
, cpu_features
, Prefer_PMINUB_for_stringop
, SSE2
, 26);
246 TUNABLE_CALLBACK (set_x86_ibt
) (tunable_val_t
*valp
)
248 if (tunable_strcmp_cte (valp
, "on"))
249 GL(dl_x86_feature_control
).ibt
= cet_always_on
;
250 else if (tunable_strcmp_cte (valp
, "off"))
251 GL(dl_x86_feature_control
).ibt
= cet_always_off
;
252 else if (tunable_strcmp_cte (valp
, "permissive"))
253 GL(dl_x86_feature_control
).ibt
= cet_permissive
;
258 TUNABLE_CALLBACK (set_x86_shstk
) (tunable_val_t
*valp
)
260 if (tunable_strcmp_cte (valp
, "on"))
261 GL(dl_x86_feature_control
).shstk
= cet_always_on
;
262 else if (tunable_strcmp_cte (valp
, "off"))
263 GL(dl_x86_feature_control
).shstk
= cet_always_off
;
264 else if (tunable_strcmp_cte (valp
, "permissive"))
265 GL(dl_x86_feature_control
).shstk
= cet_permissive
;