sysdeps/x86/cpu-tunables.c

   1 /* x86 CPU feature tuning.
   2    This file is part of the GNU C Library.
   3    Copyright (C) 2017-2023 Free Software Foundation, Inc.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <https://www.gnu.org/licenses/>.  */
  18
  19 #define TUNABLE_NAMESPACE cpu
  20 #include <stdbool.h>
  21 #include <stdint.h>
  22 #include <unistd.h>             /* Get STDOUT_FILENO for _dl_printf.  */
  23 #include <elf/dl-tunables.h>
  24 #include <string.h>
  25 #include <cpu-features.h>
  26 #include <ldsodefs.h>
  27
  28 /* We can't use IFUNC memcmp nor strlen in init_cpu_features from libc.a
  29    since IFUNC must be set up by init_cpu_features.  */
  30 #if defined USE_MULTIARCH && !defined SHARED
  31 # ifdef __x86_64__
  32 /* DEFAULT_MEMCMP by sysdeps/x86_64/memcmp-isa-default-impl.h.  */
  33 #  include <sysdeps/x86_64/memcmp-isa-default-impl.h>
  34 # else
  35 #  define DEFAULT_MEMCMP        __memcmp_ia32
  36 # endif
  37 extern __typeof (memcmp) DEFAULT_MEMCMP;
  38 #else
  39 # define DEFAULT_MEMCMP memcmp
  40 #endif
  41
  42 #define CHECK_GLIBC_IFUNC_CPU_OFF(f, cpu_features, name, len)           \
  43   _Static_assert (sizeof (#name) - 1 == len, #name " != " #len);        \
  44   if (!DEFAULT_MEMCMP (f, #name, len))                                  \
  45     {                                                                   \
  46       CPU_FEATURE_UNSET (cpu_features, name)                            \
  47       break;                                                            \
  48     }
  49
  50 /* Disable a preferred feature NAME.  We don't enable a preferred feature
  51    which isn't available.  */
  52 #define CHECK_GLIBC_IFUNC_PREFERRED_OFF(f, cpu_features, name, len)     \
  53   _Static_assert (sizeof (#name) - 1 == len, #name " != " #len);        \
  54   if (!DEFAULT_MEMCMP (f, #name, len))                                  \
  55     {                                                                   \
  56       cpu_features->preferred[index_arch_##name]                        \
  57         &= ~bit_arch_##name;                                            \
  58       break;                                                            \
  59     }
  60
  61 /* Enable/disable a preferred feature NAME.  */
  62 #define CHECK_GLIBC_IFUNC_PREFERRED_BOTH(f, cpu_features, name, \
  63                                           disable, len)                 \
  64   _Static_assert (sizeof (#name) - 1 == len, #name " != " #len);        \
  65   if (!DEFAULT_MEMCMP (f, #name, len))                                  \
  66     {                                                                   \
  67       if (disable)                                                      \
  68         cpu_features->preferred[index_arch_##name] &= ~bit_arch_##name; \
  69       else                                                              \
  70         cpu_features->preferred[index_arch_##name] |= bit_arch_##name;  \
  71       break;                                                            \
  72     }
  73
  74 /* Enable/disable a preferred feature NAME.  Enable a preferred feature
  75    only if the feature NEED is usable.  */
  76 #define CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH(f, cpu_features, name,    \
  77                                                need, disable, len)      \
  78   _Static_assert (sizeof (#name) - 1 == len, #name " != " #len);        \
  79   if (!DEFAULT_MEMCMP (f, #name, len))                                  \
  80     {                                                                   \
  81       if (disable)                                                      \
  82         cpu_features->preferred[index_arch_##name] &= ~bit_arch_##name; \
  83       else if (CPU_FEATURE_USABLE_P (cpu_features, need))               \
  84         cpu_features->preferred[index_arch_##name] |= bit_arch_##name;  \
  85       break;                                                            \
  86     }
  87
  88 attribute_hidden
  89 void
  90 TUNABLE_CALLBACK (set_hwcaps) (tunable_val_t *valp)
  91 {
  92   /* The current IFUNC selection is based on microbenchmarks in glibc.
  93      It should give the best performance for most workloads.  But other
  94      choices may have better performance for a particular workload or on
  95      the hardware which wasn't available when the selection was made.
  96      The environment variable:
  97
  98      GLIBC_TUNABLES=glibc.cpu.hwcaps=-xxx,yyy,-zzz,....
  99
 100      can be used to enable CPU/ARCH feature yyy, disable CPU/ARCH feature
 101      yyy and zzz, where the feature name is case-sensitive and has to
 102      match the ones in cpu-features.h.  It can be used by glibc developers
 103      to tune for a new processor or override the IFUNC selection to
 104      improve performance for a particular workload.
 105
 106      NOTE: the IFUNC selection may change over time.  Please check all
 107      multiarch implementations when experimenting.  */
 108
 109   const char *p = valp->strval, *c;
 110   struct cpu_features *cpu_features = &GLRO(dl_x86_cpu_features);
 111   size_t len;
 112
 113   do
 114     {
 115       const char *n;
 116       bool disable;
 117       size_t nl;
 118
 119       for (c = p; *c != ','; c++)
 120         if (*c == '\0')
 121           break;
 122
 123       len = c - p;
 124       disable = *p == '-';
 125       if (disable)
 126         {
 127           n = p + 1;
 128           nl = len - 1;
 129         }
 130       else
 131         {
 132           n = p;
 133           nl = len;
 134         }
 135       switch (nl)
 136         {
 137         default:
 138           break;
 139         case 3:
 140           if (disable)
 141             {
 142               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX, 3);
 143               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, CX8, 3);
 144               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, FMA, 3);
 145               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, HTT, 3);
 146               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, IBT, 3);
 147               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, RTM, 3);
 148             }
 149           break;
 150         case 4:
 151           if (disable)
 152             {
 153               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX2, 4);
 154               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, BMI1, 4);
 155               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, BMI2, 4);
 156               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, CMOV, 4);
 157               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, ERMS, 4);
 158               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, FMA4, 4);
 159               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE2, 4);
 160               CHECK_GLIBC_IFUNC_PREFERRED_OFF (n, cpu_features, I586, 4);
 161               CHECK_GLIBC_IFUNC_PREFERRED_OFF (n, cpu_features, I686, 4);
 162             }
 163           break;
 164         case 5:
 165           if (disable)
 166             {
 167               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, LZCNT, 5);
 168               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, MOVBE, 5);
 169               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SHSTK, 5);
 170               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSSE3, 5);
 171               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, XSAVE, 5);
 172             }
 173           break;
 174         case 6:
 175           if (disable)
 176             {
 177               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, POPCNT, 6);
 178               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE4_1, 6);
 179               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE4_2, 6);
 180               if (!DEFAULT_MEMCMP (n, "XSAVEC", 6))
 181                 {
 182                   /* Update xsave_state_size to XSAVE state size.  */
 183                   cpu_features->xsave_state_size
 184                     = cpu_features->xsave_state_full_size;
 185                   CPU_FEATURE_UNSET (cpu_features, XSAVEC);
 186                 }
 187             }
 188           break;
 189         case 7:
 190           if (disable)
 191             {
 192               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512F, 7);
 193               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, OSXSAVE, 7);
 194             }
 195           break;
 196         case 8:
 197           if (disable)
 198             {
 199               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512CD, 8);
 200               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512BW, 8);
 201               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512DQ, 8);
 202               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512ER, 8);
 203               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512PF, 8);
 204               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512VL, 8);
 205             }
 206           CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features, Slow_BSF,
 207                                             disable, 8);
 208           break;
 209         case 11:
 210             {
 211               CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
 212                                                 Prefer_ERMS,
 213                                                 disable, 11);
 214               CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
 215                                                 Prefer_FSRM,
 216                                                 disable, 11);
 217               CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH (n, cpu_features,
 218                                                      Slow_SSE4_2,
 219                                                      SSE4_2,
 220                                                      disable, 11);
 221             }
 222           break;
 223         case 15:
 224             {
 225               CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
 226                                                 Fast_Rep_String,
 227                                                 disable, 15);
 228             }
 229           break;
 230         case 16:
 231             {
 232               CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
 233                 (n, cpu_features, Prefer_No_AVX512, AVX512F,
 234                  disable, 16);
 235             }
 236           break;
 237         case 18:
 238             {
 239               CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
 240                                                 Fast_Copy_Backward,
 241                                                 disable, 18);
 242             }
 243           break;
 244         case 19:
 245             {
 246               CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
 247                                                 Fast_Unaligned_Load,
 248                                                 disable, 19);
 249               CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
 250                                                 Fast_Unaligned_Copy,
 251                                                 disable, 19);
 252             }
 253           break;
 254         case 20:
 255             {
 256               CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
 257                 (n, cpu_features, Prefer_No_VZEROUPPER, AVX, disable,
 258                  20);
 259             }
 260           break;
 261         case 23:
 262             {
 263               CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
 264                 (n, cpu_features, AVX_Fast_Unaligned_Load, AVX,
 265                  disable, 23);
 266             }
 267           break;
 268         case 24:
 269             {
 270               CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
 271                 (n, cpu_features, MathVec_Prefer_No_AVX512, AVX512F,
 272                  disable, 24);
 273             }
 274           break;
 275         case 26:
 276             {
 277               CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
 278                 (n, cpu_features, Prefer_PMINUB_for_stringop, SSE2,
 279                  disable, 26);
 280             }
 281           break;
 282         }
 283       p += len + 1;
 284     }
 285   while (*c != '\0');
 286 }
 287
 288 #if CET_ENABLED
 289 attribute_hidden
 290 void
 291 TUNABLE_CALLBACK (set_x86_ibt) (tunable_val_t *valp)
 292 {
 293   if (DEFAULT_MEMCMP (valp->strval, "on", sizeof ("on")) == 0)
 294     GL(dl_x86_feature_control).ibt = cet_always_on;
 295   else if (DEFAULT_MEMCMP (valp->strval, "off", sizeof ("off")) == 0)
 296     GL(dl_x86_feature_control).ibt = cet_always_off;
 297   else if (DEFAULT_MEMCMP (valp->strval, "permissive",
 298                            sizeof ("permissive")) == 0)
 299     GL(dl_x86_feature_control).ibt = cet_permissive;
 300 }
 301
 302 attribute_hidden
 303 void
 304 TUNABLE_CALLBACK (set_x86_shstk) (tunable_val_t *valp)
 305 {
 306   if (DEFAULT_MEMCMP (valp->strval, "on", sizeof ("on")) == 0)
 307     GL(dl_x86_feature_control).shstk = cet_always_on;
 308   else if (DEFAULT_MEMCMP (valp->strval, "off", sizeof ("off")) == 0)
 309     GL(dl_x86_feature_control).shstk = cet_always_off;
 310   else if (DEFAULT_MEMCMP (valp->strval, "permissive",
 311                            sizeof ("permissive")) == 0)
 312     GL(dl_x86_feature_control).shstk = cet_permissive;
 313 }
 314 #endif