sysdeps/x86/cpu-tunables.c

   1 /* x86 CPU feature tuning.
   2    This file is part of the GNU C Library.
   3    Copyright (C) 2017-2021 Free Software Foundation, Inc.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <https://www.gnu.org/licenses/>.  */
  18
  19 #if HAVE_TUNABLES
  20 # define TUNABLE_NAMESPACE cpu
  21 # include <stdbool.h>
  22 # include <stdint.h>
  23 # include <unistd.h>            /* Get STDOUT_FILENO for _dl_printf.  */
  24 # include <elf/dl-tunables.h>
  25 # include <string.h>
  26 # include <cpu-features.h>
  27 # include <ldsodefs.h>
  28
  29 /* We can't use IFUNC memcmp nor strlen in init_cpu_features from libc.a
  30    since IFUNC must be set up by init_cpu_features.  */
  31 # if defined USE_MULTIARCH && !defined SHARED
  32 #  ifdef __x86_64__
  33 #   define DEFAULT_MEMCMP       __memcmp_sse2
  34 #  else
  35 #   define DEFAULT_MEMCMP       __memcmp_ia32
  36 #  endif
  37 extern __typeof (memcmp) DEFAULT_MEMCMP;
  38 # else
  39 #  define DEFAULT_MEMCMP        memcmp
  40 # endif
  41
  42 # define CHECK_GLIBC_IFUNC_CPU_OFF(f, cpu_features, name, len)          \
  43   _Static_assert (sizeof (#name) - 1 == len, #name " != " #len);        \
  44   if (!DEFAULT_MEMCMP (f, #name, len))                                  \
  45     {                                                                   \
  46       CPU_FEATURE_UNSET (cpu_features, name)                            \
  47       break;                                                            \
  48     }
  49
  50 /* Disable a preferred feature NAME.  We don't enable a preferred feature
  51    which isn't available.  */
  52 # define CHECK_GLIBC_IFUNC_PREFERRED_OFF(f, cpu_features, name, len)    \
  53   _Static_assert (sizeof (#name) - 1 == len, #name " != " #len);        \
  54   if (!DEFAULT_MEMCMP (f, #name, len))                                  \
  55     {                                                                   \
  56       cpu_features->preferred[index_arch_##name]                        \
  57         &= ~bit_arch_##name;                                            \
  58       break;                                                            \
  59     }
  60
  61 /* Enable/disable a preferred feature NAME.  */
  62 # define CHECK_GLIBC_IFUNC_PREFERRED_BOTH(f, cpu_features, name,        \
  63                                           disable, len)                 \
  64   _Static_assert (sizeof (#name) - 1 == len, #name " != " #len);        \
  65   if (!DEFAULT_MEMCMP (f, #name, len))                                  \
  66     {                                                                   \
  67       if (disable)                                                      \
  68         cpu_features->preferred[index_arch_##name] &= ~bit_arch_##name; \
  69       else                                                              \
  70         cpu_features->preferred[index_arch_##name] |= bit_arch_##name;  \
  71       break;                                                            \
  72     }
  73
  74 /* Enable/disable a preferred feature NAME.  Enable a preferred feature
  75    only if the feature NEED is usable.  */
  76 # define CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH(f, cpu_features, name,   \
  77                                                need, disable, len)      \
  78   _Static_assert (sizeof (#name) - 1 == len, #name " != " #len);        \
  79   if (!DEFAULT_MEMCMP (f, #name, len))                                  \
  80     {                                                                   \
  81       if (disable)                                                      \
  82         cpu_features->preferred[index_arch_##name] &= ~bit_arch_##name; \
  83       else if (CPU_FEATURE_USABLE_P (cpu_features, need))               \
  84         cpu_features->preferred[index_arch_##name] |= bit_arch_##name;  \
  85       break;                                                            \
  86     }
  87
  88 attribute_hidden
  89 void
  90 TUNABLE_CALLBACK (set_hwcaps) (tunable_val_t *valp)
  91 {
  92   /* The current IFUNC selection is based on microbenchmarks in glibc.
  93      It should give the best performance for most workloads.  But other
  94      choices may have better performance for a particular workload or on
  95      the hardware which wasn't available when the selection was made.
  96      The environment variable:
  97
  98      GLIBC_TUNABLES=glibc.cpu.hwcaps=-xxx,yyy,-zzz,....
  99
 100      can be used to enable CPU/ARCH feature yyy, disable CPU/ARCH feature
 101      yyy and zzz, where the feature name is case-sensitive and has to
 102      match the ones in cpu-features.h.  It can be used by glibc developers
 103      to tune for a new processor or override the IFUNC selection to
 104      improve performance for a particular workload.
 105
 106      NOTE: the IFUNC selection may change over time.  Please check all
 107      multiarch implementations when experimenting.  */
 108
 109   const char *p = valp->strval;
 110   struct cpu_features *cpu_features = &GLRO(dl_x86_cpu_features);
 111   size_t len;
 112
 113   do
 114     {
 115       const char *c, *n;
 116       bool disable;
 117       size_t nl;
 118
 119       for (c = p; *c != ','; c++)
 120         if (*c == '\0')
 121           break;
 122
 123       len = c - p;
 124       disable = *p == '-';
 125       if (disable)
 126         {
 127           n = p + 1;
 128           nl = len - 1;
 129         }
 130       else
 131         {
 132           n = p;
 133           nl = len;
 134         }
 135       switch (nl)
 136         {
 137         default:
 138           break;
 139         case 3:
 140           if (disable)
 141             {
 142               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX, 3);
 143               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, CX8, 3);
 144               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, FMA, 3);
 145               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, HTT, 3);
 146               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, IBT, 3);
 147               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, RTM, 3);
 148             }
 149           break;
 150         case 4:
 151           if (disable)
 152             {
 153               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX2, 4);
 154               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, BMI1, 4);
 155               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, BMI2, 4);
 156               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, CMOV, 4);
 157               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, ERMS, 4);
 158               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, FMA4, 4);
 159               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE2, 4);
 160               CHECK_GLIBC_IFUNC_PREFERRED_OFF (n, cpu_features, I586, 4);
 161               CHECK_GLIBC_IFUNC_PREFERRED_OFF (n, cpu_features, I686, 4);
 162             }
 163           break;
 164         case 5:
 165           if (disable)
 166             {
 167               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, LZCNT, 5);
 168               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, MOVBE, 5);
 169               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SHSTK, 5);
 170               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSSE3, 5);
 171             }
 172           break;
 173         case 6:
 174           if (disable)
 175             {
 176               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, POPCNT, 6);
 177               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE4_1, 6);
 178               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, SSE4_2, 6);
 179               if (!DEFAULT_MEMCMP (n, "XSAVEC", 6))
 180                 {
 181                   /* Update xsave_state_size to XSAVE state size.  */
 182                   cpu_features->xsave_state_size
 183                     = cpu_features->xsave_state_full_size;
 184                   CPU_FEATURE_UNSET (cpu_features, XSAVEC);
 185                 }
 186             }
 187           break;
 188         case 7:
 189           if (disable)
 190             {
 191               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512F, 7);
 192               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, OSXSAVE, 7);
 193             }
 194           break;
 195         case 8:
 196           if (disable)
 197             {
 198               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512CD, 8);
 199               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512BW, 8);
 200               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512DQ, 8);
 201               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512ER, 8);
 202               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512PF, 8);
 203               CHECK_GLIBC_IFUNC_CPU_OFF (n, cpu_features, AVX512VL, 8);
 204             }
 205           CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features, Slow_BSF,
 206                                             disable, 8);
 207           break;
 208         case 11:
 209             {
 210               CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
 211                                                 Prefer_ERMS,
 212                                                 disable, 11);
 213               CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
 214                                                 Prefer_FSRM,
 215                                                 disable, 11);
 216               CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH (n, cpu_features,
 217                                                      Slow_SSE4_2,
 218                                                      SSE4_2,
 219                                                      disable, 11);
 220             }
 221           break;
 222         case 15:
 223             {
 224               CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
 225                                                 Fast_Rep_String,
 226                                                 disable, 15);
 227             }
 228           break;
 229         case 16:
 230             {
 231               CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
 232                 (n, cpu_features, Prefer_No_AVX512, AVX512F,
 233                  disable, 16);
 234             }
 235           break;
 236         case 18:
 237             {
 238               CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
 239                                                 Fast_Copy_Backward,
 240                                                 disable, 18);
 241             }
 242           break;
 243         case 19:
 244             {
 245               CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
 246                                                 Fast_Unaligned_Load,
 247                                                 disable, 19);
 248               CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
 249                                                 Fast_Unaligned_Copy,
 250                                                 disable, 19);
 251             }
 252           break;
 253         case 20:
 254             {
 255               CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
 256                 (n, cpu_features, Prefer_No_VZEROUPPER, AVX, disable,
 257                  20);
 258             }
 259           break;
 260         case 21:
 261             {
 262               CHECK_GLIBC_IFUNC_PREFERRED_BOTH (n, cpu_features,
 263                                                 Prefer_MAP_32BIT_EXEC,
 264                                                 disable, 21);
 265             }
 266           break;
 267         case 23:
 268             {
 269               CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
 270                 (n, cpu_features, AVX_Fast_Unaligned_Load, AVX,
 271                  disable, 23);
 272             }
 273           break;
 274         case 24:
 275             {
 276               CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
 277                 (n, cpu_features, MathVec_Prefer_No_AVX512, AVX512F,
 278                  disable, 24);
 279             }
 280           break;
 281         case 26:
 282             {
 283               CHECK_GLIBC_IFUNC_PREFERRED_NEED_BOTH
 284                 (n, cpu_features, Prefer_PMINUB_for_stringop, SSE2,
 285                  disable, 26);
 286             }
 287           break;
 288         }
 289       p += len + 1;
 290     }
 291   while (*p != '\0');
 292 }
 293
 294 # if CET_ENABLED
 295
 296 attribute_hidden
 297 void
 298 TUNABLE_CALLBACK (set_x86_ibt) (tunable_val_t *valp)
 299 {
 300   if (DEFAULT_MEMCMP (valp->strval, "on", sizeof ("on")) == 0)
 301     GL(dl_x86_feature_control).ibt = cet_always_on;
 302   else if (DEFAULT_MEMCMP (valp->strval, "off", sizeof ("off")) == 0)
 303     GL(dl_x86_feature_control).ibt = cet_always_off;
 304   else if (DEFAULT_MEMCMP (valp->strval, "permissive",
 305                            sizeof ("permissive")) == 0)
 306     GL(dl_x86_feature_control).ibt = cet_permissive;
 307 }
 308
 309 attribute_hidden
 310 void
 311 TUNABLE_CALLBACK (set_x86_shstk) (tunable_val_t *valp)
 312 {
 313   if (DEFAULT_MEMCMP (valp->strval, "on", sizeof ("on")) == 0)
 314     GL(dl_x86_feature_control).shstk = cet_always_on;
 315   else if (DEFAULT_MEMCMP (valp->strval, "off", sizeof ("off")) == 0)
 316     GL(dl_x86_feature_control).shstk = cet_always_off;
 317   else if (DEFAULT_MEMCMP (valp->strval, "permissive",
 318                            sizeof ("permissive")) == 0)
 319     GL(dl_x86_feature_control).shstk = cet_permissive;
 320 }
 321 # endif
 322 #endif