sysdeps/x86/cpu-tunables.c

   1 /* x86 CPU feature tuning.
   2    This file is part of the GNU C Library.
   3    Copyright (C) 2017-2023 Free Software Foundation, Inc.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <https://www.gnu.org/licenses/>.  */
  18
  19 #if HAVE_TUNABLES
  20 # define TUNABLE_NAMESPACE cpu
  21 # include <stdbool.h>
  22 # include <stdint.h>
  23 # include <unistd.h>            /* Get STDOUT_FILENO for _dl_printf.  */
  24 # include <elf/dl-tunables.h>
  25 # include <string.h>
  26 # include <cpu-features.h>
  27 # include <ldsodefs.h>
  28
  29 /* We can't use IFUNC memcmp nor strlen in init_cpu_features from libc.a
  30    since IFUNC must be set up by init_cpu_features.  */
  31 # if defined USE_MULTIARCH && !defined SHARED
  32 #  ifdef __x86_64__
  33 /* DEFAULT_MEMCMP by sysdeps/x86_64/memcmp-isa-default-impl.h.  */
  34 #   include <sysdeps/x86_64/memcmp-isa-default-impl.h>
  35 #  else
  36 #   define DEFAULT_MEMCMP       __memcmp_ia32
  37 #  endif
  38 extern __typeof (memcmp) DEFAULT_MEMCMP;
  39 # else
  40 #  define DEFAULT_MEMCMP        memcmp
  41 # endif
  42
  43 # define CHECK_GLIBC_IFUNC_CPU_OFF(f, cpu_features, name, len)          \
  44   _Static_assert (sizeof (#name) - 1 == len, #name " != " #len);        \
  45   if (!DEFAULT_MEMCMP (f, #name, len))                                  \
  46     {                                                                   \
  47       CPU_FEATURE_UNSET (cpu_features, name)                            \
  48       break;                                                            \
  49     }
  50
  51 /* Disable a preferred feature NAME.  We don't enable a preferred feature
  52    which isn't available.  */
  53 # define CHECK_GLIBC_IFUNC_PREFERRED_OFF(f, cpu_features, name, len)    \
  54   _Static_assert (sizeof (#name) - 1 == len, #name " != " #len);        \
  55   if (!DEFAULT_MEMCMP (f, #name, len))                                  \
  56     {                                                                   \
  57       cpu_features->preferred[index_arch_##name]                        \
  58         &= ~bit_arch_##name;                                            \
  59       break;                                                            \
  60     }
  61
  62 /* Enable/disable a preferred feature NAME.  */
  63 # define CHECK_GLIBC_IFUNC_PREFERRED_BOTH(f, cpu_features, name,        \
  64                                           disable, len)                 \
  65   _Static_assert (sizeof (#name) - 1 == len, #name " != " #len);        \
  66   if (!DEFAULT_MEMCMP (f, #name, len))                                  \
  67     {                                                                   \
  68       if (disable)                                                      \
  69         cpu_features->preferred[index_arch_##name] &= ~bit_arch_##name; \
  70       else                                                              \
  71         cpu_features->preferred[index_arch_##name] |= bit_arch_##name;  \
  72       break;                                                            \
  73     }
  74
  75 /* Enable/disable a preferred feature NAME.  Enable a preferred feature
  76    only if the feature NEED is usable.  */
  77 # define CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH(f, cpu_features, name,   \
  78                                                need, disable, len)      \
  79   _Static_assert (sizeof (#name) - 1 == len, #name " != " #len);        \
  80   if (!DEFAULT_MEMCMP (f, #name, len))                                  \
  81     {                                                                   \
  82       if (disable)                                                      \
  83         cpu_features->preferred[index_arch_##name] &= ~bit_arch_##name; \
  84       else if (CPU_FEATURE_USABLE_P (cpu_features, need))               \
  85         cpu_features->preferred[index_arch_##name] |= bit_arch_##name;  \
  86       break;                                                            \
  87     }
  88
  89 attribute_hidden
  90 void
  91 TUNABLE_CALLBACK (set_hwcaps) (tunable_val_t *valp)
  92 {
  93   /* The current IFUNC selection is based on microbenchmarks in glibc.
  94      It should give the best performance for most workloads.  But other
  95      choices may have better performance for a particular workload or on
  96      the hardware which wasn't available when the selection was made.
  97      The environment variable:
  98
  99      GLIBC_TUNABLES=glibc.cpu.hwcaps=-xxx,yyy,-zzz,....
 100
 101      can be used to enable CPU/ARCH feature yyy, disable CPU/ARCH feature
 102      yyy and zzz, where the feature name is case-sensitive and has to
 103      match the ones in cpu-features.h.  It can be used by glibc developers
 104      to tune for a new processor or override the IFUNC selection to
 105      improve performance for a particular workload.
 106
 107      NOTE: the IFUNC selection may change over time.  Please check all
 108      multiarch implementations when experimenting.  */
 109
 110   const char *p = valp->strval, *c;
 111   struct cpu_features *cpu_features = &GLRO(dl_x86_cpu_features);
 112   size_t len;
 113
 114   do
 115     {
 116       const char *n;
 117       bool disable;
 118       size_t nl;
 119
 120       for (c = p; *c != ','; c++)
 121         if (*c == '\0')
 122           break;
 123
 124       len = c - p;
 125       disable = *p == '-';
 126       if (disable)
 127         {
 128           n = p + 1;
 129           nl = len - 1;
 130         }
 131       else
 132         {
 133           n = p;
 134           nl = len;
 135         }
 136       switch (nl)
 137         {
 138         default:
 139           break;
 140         case 3:
 141           if (disable)
 142             {
 143               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX, 3);
 144               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, CX8, 3);
 145               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, FMA, 3);
 146               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, HTT, 3);
 147               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, IBT, 3);
 148               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, RTM, 3);
 149             }
 150           break;
 151         case 4:
 152           if (disable)
 153             {
 154               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX2, 4);
 155               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, BMI1, 4);
 156               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, BMI2, 4);
 157               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, CMOV, 4);
 158               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, ERMS, 4);
 159               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, FMA4, 4);
 160               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE2, 4);
 161               CHECK_GLIBC_IFUNC_PREFERRED_OFF (n, cpu_features, I586, 4);
 162               CHECK_GLIBC_IFUNC_PREFERRED_OFF (n, cpu_features, I686, 4);
 163             }
 164           break;
 165         case 5:
 166           if (disable)
 167             {
 168               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, LZCNT, 5);
 169               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, MOVBE, 5);
 170               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SHSTK, 5);
 171               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSSE3, 5);
 172               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, XSAVE, 5);
 173             }
 174           break;
 175         case 6:
 176           if (disable)
 177             {
 178               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, POPCNT, 6);
 179               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE4_1, 6);
 180               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE4_2, 6);
 181               if (!DEFAULT_MEMCMP (n, "XSAVEC", 6))
 182                 {
 183                   /* Update xsave_state_size to XSAVE state size.  */
 184                   cpu_features->xsave_state_size
 185                     = cpu_features->xsave_state_full_size;
 186                   CPU_FEATURE_UNSET (cpu_features, XSAVEC);
 187                 }
 188             }
 189           break;
 190         case 7:
 191           if (disable)
 192             {
 193               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512F, 7);
 194               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, OSXSAVE, 7);
 195             }
 196           break;
 197         case 8:
 198           if (disable)
 199             {
 200               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512CD, 8);
 201               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512BW, 8);
 202               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512DQ, 8);
 203               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512ER, 8);
 204               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512PF, 8);
 205               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512VL, 8);
 206             }
 207           CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features, Slow_BSF,
 208                                             disable, 8);
 209           break;
 210         case 11:
 211             {
 212               CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
 213                                                 Prefer_ERMS,
 214                                                 disable, 11);
 215               CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
 216                                                 Prefer_FSRM,
 217                                                 disable, 11);
 218               CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH (n, cpu_features,
 219                                                      Slow_SSE4_2,
 220                                                      SSE4_2,
 221                                                      disable, 11);
 222             }
 223           break;
 224         case 15:
 225             {
 226               CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
 227                                                 Fast_Rep_String,
 228                                                 disable, 15);
 229             }
 230           break;
 231         case 16:
 232             {
 233               CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
 234                 (n, cpu_features, Prefer_No_AVX512, AVX512F,
 235                  disable, 16);
 236             }
 237           break;
 238         case 18:
 239             {
 240               CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
 241                                                 Fast_Copy_Backward,
 242                                                 disable, 18);
 243             }
 244           break;
 245         case 19:
 246             {
 247               CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
 248                                                 Fast_Unaligned_Load,
 249                                                 disable, 19);
 250               CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
 251                                                 Fast_Unaligned_Copy,
 252                                                 disable, 19);
 253             }
 254           break;
 255         case 20:
 256             {
 257               CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
 258                 (n, cpu_features, Prefer_No_VZEROUPPER, AVX, disable,
 259                  20);
 260             }
 261           break;
 262         case 23:
 263             {
 264               CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
 265                 (n, cpu_features, AVX_Fast_Unaligned_Load, AVX,
 266                  disable, 23);
 267             }
 268           break;
 269         case 24:
 270             {
 271               CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
 272                 (n, cpu_features, MathVec_Prefer_No_AVX512, AVX512F,
 273                  disable, 24);
 274             }
 275           break;
 276         case 26:
 277             {
 278               CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
 279                 (n, cpu_features, Prefer_PMINUB_for_stringop, SSE2,
 280                  disable, 26);
 281             }
 282           break;
 283         }
 284       p += len + 1;
 285     }
 286   while (*c != '\0');
 287 }
 288
 289 # if CET_ENABLED
 290
 291 attribute_hidden
 292 void
 293 TUNABLE_CALLBACK (set_x86_ibt) (tunable_val_t *valp)
 294 {
 295   if (DEFAULT_MEMCMP (valp->strval, "on", sizeof ("on")) == 0)
 296     GL(dl_x86_feature_control).ibt = cet_always_on;
 297   else if (DEFAULT_MEMCMP (valp->strval, "off", sizeof ("off")) == 0)
 298     GL(dl_x86_feature_control).ibt = cet_always_off;
 299   else if (DEFAULT_MEMCMP (valp->strval, "permissive",
 300                            sizeof ("permissive")) == 0)
 301     GL(dl_x86_feature_control).ibt = cet_permissive;
 302 }
 303
 304 attribute_hidden
 305 void
 306 TUNABLE_CALLBACK (set_x86_shstk) (tunable_val_t *valp)
 307 {
 308   if (DEFAULT_MEMCMP (valp->strval, "on", sizeof ("on")) == 0)
 309     GL(dl_x86_feature_control).shstk = cet_always_on;
 310   else if (DEFAULT_MEMCMP (valp->strval, "off", sizeof ("off")) == 0)
 311     GL(dl_x86_feature_control).shstk = cet_always_off;
 312   else if (DEFAULT_MEMCMP (valp->strval, "permissive",
 313                            sizeof ("permissive")) == 0)
 314     GL(dl_x86_feature_control).shstk = cet_permissive;
 315 }
 316 # endif
 317 #endif