sysdeps/x86/cpu-features.c

   1 /* Initialize CPU feature data.
   2    This file is part of the GNU C Library.
   3    Copyright (C) 2008-2017 Free Software Foundation, Inc.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <http://www.gnu.org/licenses/>.  */
  18
  19 #include <cpuid.h>
  20 #include <cpu-features.h>
  21 #include <dl-hwcap.h>
  22
  23 #if HAVE_TUNABLES
  24 # define TUNABLE_NAMESPACE tune
  25 # include <unistd.h>            /* Get STDOUT_FILENO for _dl_printf.  */
  26 # include <elf/dl-tunables.h>
  27
  28 extern void TUNABLE_CALLBACK (set_hwcaps) (tunable_val_t *)
  29   attribute_hidden;
  30 #endif
  31
  32 static void
  33 get_common_indeces (struct cpu_features *cpu_features,
  34                     unsigned int *family, unsigned int *model,
  35                     unsigned int *extended_model, unsigned int *stepping)
  36 {
  37   if (family)
  38     {
  39       unsigned int eax;
  40       __cpuid (1, eax, cpu_features->cpuid[COMMON_CPUID_INDEX_1].ebx,
  41                cpu_features->cpuid[COMMON_CPUID_INDEX_1].ecx,
  42                cpu_features->cpuid[COMMON_CPUID_INDEX_1].edx);
  43       cpu_features->cpuid[COMMON_CPUID_INDEX_1].eax = eax;
  44       *family = (eax >> 8) & 0x0f;
  45       *model = (eax >> 4) & 0x0f;
  46       *extended_model = (eax >> 12) & 0xf0;
  47       *stepping = eax & 0x0f;
  48       if (*family == 0x0f)
  49         {
  50           *family += (eax >> 20) & 0xff;
  51           *model += *extended_model;
  52         }
  53     }
  54
  55   if (cpu_features->max_cpuid >= 7)
  56     __cpuid_count (7, 0,
  57                    cpu_features->cpuid[COMMON_CPUID_INDEX_7].eax,
  58                    cpu_features->cpuid[COMMON_CPUID_INDEX_7].ebx,
  59                    cpu_features->cpuid[COMMON_CPUID_INDEX_7].ecx,
  60                    cpu_features->cpuid[COMMON_CPUID_INDEX_7].edx);
  61
  62   /* Can we call xgetbv?  */
  63   if (CPU_FEATURES_CPU_P (cpu_features, OSXSAVE))
  64     {
  65       unsigned int xcrlow;
  66       unsigned int xcrhigh;
  67       asm ("xgetbv" : "=a" (xcrlow), "=d" (xcrhigh) : "c" (0));
  68       /* Is YMM and XMM state usable?  */
  69       if ((xcrlow & (bit_YMM_state | bit_XMM_state)) ==
  70           (bit_YMM_state | bit_XMM_state))
  71         {
  72           /* Determine if AVX is usable.  */
  73           if (CPU_FEATURES_CPU_P (cpu_features, AVX))
  74             {
  75               cpu_features->feature[index_arch_AVX_Usable]
  76                 |= bit_arch_AVX_Usable;
  77               /* The following features depend on AVX being usable.  */
  78               /* Determine if AVX2 is usable.  */
  79               if (CPU_FEATURES_CPU_P (cpu_features, AVX2))
  80                 cpu_features->feature[index_arch_AVX2_Usable]
  81                   |= bit_arch_AVX2_Usable;
  82               /* Determine if FMA is usable.  */
  83               if (CPU_FEATURES_CPU_P (cpu_features, FMA))
  84                 cpu_features->feature[index_arch_FMA_Usable]
  85                   |= bit_arch_FMA_Usable;
  86             }
  87
  88           /* Check if OPMASK state, upper 256-bit of ZMM0-ZMM15 and
  89              ZMM16-ZMM31 state are enabled.  */
  90           if ((xcrlow & (bit_Opmask_state | bit_ZMM0_15_state
  91                          | bit_ZMM16_31_state)) ==
  92               (bit_Opmask_state | bit_ZMM0_15_state | bit_ZMM16_31_state))
  93             {
  94               /* Determine if AVX512F is usable.  */
  95               if (CPU_FEATURES_CPU_P (cpu_features, AVX512F))
  96                 {
  97                   cpu_features->feature[index_arch_AVX512F_Usable]
  98                     |= bit_arch_AVX512F_Usable;
  99                   /* Determine if AVX512DQ is usable.  */
 100                   if (CPU_FEATURES_CPU_P (cpu_features, AVX512DQ))
 101                     cpu_features->feature[index_arch_AVX512DQ_Usable]
 102                       |= bit_arch_AVX512DQ_Usable;
 103                 }
 104             }
 105         }
 106     }
 107 }
 108
 109 static inline void
 110 init_cpu_features (struct cpu_features *cpu_features)
 111 {
 112   unsigned int ebx, ecx, edx;
 113   unsigned int family = 0;
 114   unsigned int model = 0;
 115   enum cpu_features_kind kind;
 116
 117 #if !HAS_CPUID
 118   if (__get_cpuid_max (0, 0) == 0)
 119     {
 120       kind = arch_kind_other;
 121       goto no_cpuid;
 122     }
 123 #endif
 124
 125   __cpuid (0, cpu_features->max_cpuid, ebx, ecx, edx);
 126
 127   /* This spells out "GenuineIntel".  */
 128   if (ebx == 0x756e6547 && ecx == 0x6c65746e && edx == 0x49656e69)
 129     {
 130       unsigned int extended_model, stepping;
 131
 132       kind = arch_kind_intel;
 133
 134       get_common_indeces (cpu_features, &family, &model, &extended_model,
 135                           &stepping);
 136
 137       if (family == 0x06)
 138         {
 139           model += extended_model;
 140           switch (model)
 141             {
 142             case 0x1c:
 143             case 0x26:
 144               /* BSF is slow on Atom.  */
 145               cpu_features->feature[index_arch_Slow_BSF]
 146                 |= bit_arch_Slow_BSF;
 147               break;
 148
 149             case 0x57:
 150               /* Knights Landing.  Enable Silvermont optimizations.  */
 151
 152             case 0x5c:
 153             case 0x5f:
 154               /* Unaligned load versions are faster than SSSE3
 155                  on Goldmont.  */
 156
 157             case 0x4c:
 158               /* Airmont is a die shrink of Silvermont.  */
 159
 160             case 0x37:
 161             case 0x4a:
 162             case 0x4d:
 163             case 0x5a:
 164             case 0x5d:
 165               /* Unaligned load versions are faster than SSSE3
 166                  on Silvermont.  */
 167 #if index_arch_Fast_Unaligned_Load != index_arch_Prefer_PMINUB_for_stringop
 168 # error index_arch_Fast_Unaligned_Load != index_arch_Prefer_PMINUB_for_stringop
 169 #endif
 170 #if index_arch_Fast_Unaligned_Load != index_arch_Slow_SSE4_2
 171 # error index_arch_Fast_Unaligned_Load != index_arch_Slow_SSE4_2
 172 #endif
 173 #if index_arch_Fast_Unaligned_Load != index_arch_Fast_Unaligned_Copy
 174 # error index_arch_Fast_Unaligned_Load != index_arch_Fast_Unaligned_Copy
 175 #endif
 176               cpu_features->feature[index_arch_Fast_Unaligned_Load]
 177                 |= (bit_arch_Fast_Unaligned_Load
 178                     | bit_arch_Fast_Unaligned_Copy
 179                     | bit_arch_Prefer_PMINUB_for_stringop
 180                     | bit_arch_Slow_SSE4_2);
 181               break;
 182
 183             default:
 184               /* Unknown family 0x06 processors.  Assuming this is one
 185                  of Core i3/i5/i7 processors if AVX is available.  */
 186               if (!CPU_FEATURES_CPU_P (cpu_features, AVX))
 187                 break;
 188
 189             case 0x1a:
 190             case 0x1e:
 191             case 0x1f:
 192             case 0x25:
 193             case 0x2c:
 194             case 0x2e:
 195             case 0x2f:
 196               /* Rep string instructions, unaligned load, unaligned copy,
 197                  and pminub are fast on Intel Core i3, i5 and i7.  */
 198 #if index_arch_Fast_Rep_String != index_arch_Fast_Unaligned_Load
 199 # error index_arch_Fast_Rep_String != index_arch_Fast_Unaligned_Load
 200 #endif
 201 #if index_arch_Fast_Rep_String != index_arch_Prefer_PMINUB_for_stringop
 202 # error index_arch_Fast_Rep_String != index_arch_Prefer_PMINUB_for_stringop
 203 #endif
 204 #if index_arch_Fast_Rep_String != index_arch_Fast_Unaligned_Copy
 205 # error index_arch_Fast_Rep_String != index_arch_Fast_Unaligned_Copy
 206 #endif
 207               cpu_features->feature[index_arch_Fast_Rep_String]
 208                 |= (bit_arch_Fast_Rep_String
 209                     | bit_arch_Fast_Unaligned_Load
 210                     | bit_arch_Fast_Unaligned_Copy
 211                     | bit_arch_Prefer_PMINUB_for_stringop);
 212               break;
 213
 214             case 0x3f:
 215               /* Xeon E7 v3 with stepping >= 4 has working TSX.  */
 216               if (stepping >= 4)
 217                 break;
 218             case 0x3c:
 219             case 0x45:
 220             case 0x46:
 221               /* Disable Intel TSX on Haswell processors (except Xeon E7 v3
 222                  with stepping >= 4) to avoid TSX on kernels that weren't
 223                  updated with the latest microcode package (which disables
 224                  broken feature by default).  */
 225               cpu_features->cpuid[index_cpu_RTM].reg_RTM &= ~bit_cpu_RTM;
 226               break;
 227             }
 228         }
 229
 230       /* Unaligned load with 256-bit AVX registers are faster on
 231          Intel processors with AVX2.  */
 232       if (CPU_FEATURES_ARCH_P (cpu_features, AVX2_Usable))
 233         cpu_features->feature[index_arch_AVX_Fast_Unaligned_Load]
 234           |= bit_arch_AVX_Fast_Unaligned_Load;
 235
 236       /* Since AVX512ER is unique to Xeon Phi, set Prefer_No_VZEROUPPER
 237          if AVX512ER is available.  Don't use AVX512 to avoid lower CPU
 238          frequency if AVX512ER isn't available.  */
 239       if (CPU_FEATURES_CPU_P (cpu_features, AVX512ER))
 240         cpu_features->feature[index_arch_Prefer_No_VZEROUPPER]
 241           |= bit_arch_Prefer_No_VZEROUPPER;
 242       else
 243         cpu_features->feature[index_arch_Prefer_No_AVX512]
 244           |= bit_arch_Prefer_No_AVX512;
 245
 246       /* To avoid SSE transition penalty, use _dl_runtime_resolve_slow.
 247          If XGETBV suports ECX == 1, use _dl_runtime_resolve_opt.
 248          Use _dl_runtime_resolve_opt only with AVX512F since it is
 249          slower than _dl_runtime_resolve_slow with AVX.  */
 250       cpu_features->feature[index_arch_Use_dl_runtime_resolve_slow]
 251         |= bit_arch_Use_dl_runtime_resolve_slow;
 252       if (CPU_FEATURES_ARCH_P (cpu_features, AVX512F_Usable)
 253           && cpu_features->max_cpuid >= 0xd)
 254         {
 255           unsigned int eax;
 256
 257           __cpuid_count (0xd, 1, eax, ebx, ecx, edx);
 258           if ((eax & (1 << 2)) != 0)
 259             cpu_features->feature[index_arch_Use_dl_runtime_resolve_opt]
 260               |= bit_arch_Use_dl_runtime_resolve_opt;
 261         }
 262     }
 263   /* This spells out "AuthenticAMD".  */
 264   else if (ebx == 0x68747541 && ecx == 0x444d4163 && edx == 0x69746e65)
 265     {
 266       unsigned int extended_model, stepping;
 267
 268       kind = arch_kind_amd;
 269
 270       get_common_indeces (cpu_features, &family, &model, &extended_model,
 271                           &stepping);
 272
 273       ecx = cpu_features->cpuid[COMMON_CPUID_INDEX_1].ecx;
 274
 275       unsigned int eax;
 276       __cpuid (0x80000000, eax, ebx, ecx, edx);
 277       if (eax >= 0x80000001)
 278         __cpuid (0x80000001,
 279                  cpu_features->cpuid[COMMON_CPUID_INDEX_80000001].eax,
 280                  cpu_features->cpuid[COMMON_CPUID_INDEX_80000001].ebx,
 281                  cpu_features->cpuid[COMMON_CPUID_INDEX_80000001].ecx,
 282                  cpu_features->cpuid[COMMON_CPUID_INDEX_80000001].edx);
 283
 284       if (HAS_ARCH_FEATURE (AVX_Usable))
 285         {
 286           /* Since the FMA4 bit is in COMMON_CPUID_INDEX_80000001 and
 287              FMA4 requires AVX, determine if FMA4 is usable here.  */
 288           if (CPU_FEATURES_CPU_P (cpu_features, FMA4))
 289             cpu_features->feature[index_arch_FMA4_Usable]
 290               |= bit_arch_FMA4_Usable;
 291         }
 292
 293       if (family == 0x15)
 294         {
 295 #if index_arch_Fast_Unaligned_Load != index_arch_Fast_Copy_Backward
 296 # error index_arch_Fast_Unaligned_Load != index_arch_Fast_Copy_Backward
 297 #endif
 298           /* "Excavator"   */
 299           if (model >= 0x60 && model <= 0x7f)
 300             cpu_features->feature[index_arch_Fast_Unaligned_Load]
 301               |= (bit_arch_Fast_Unaligned_Load
 302                   | bit_arch_Fast_Copy_Backward);
 303         }
 304     }
 305   else
 306     {
 307       kind = arch_kind_other;
 308       get_common_indeces (cpu_features, NULL, NULL, NULL, NULL);
 309     }
 310
 311   /* Support i586 if CX8 is available.  */
 312   if (CPU_FEATURES_CPU_P (cpu_features, CX8))
 313     cpu_features->feature[index_arch_I586] |= bit_arch_I586;
 314
 315   /* Support i686 if CMOV is available.  */
 316   if (CPU_FEATURES_CPU_P (cpu_features, CMOV))
 317     cpu_features->feature[index_arch_I686] |= bit_arch_I686;
 318
 319 #if !HAS_CPUID
 320 no_cpuid:
 321 #endif
 322
 323   cpu_features->family = family;
 324   cpu_features->model = model;
 325   cpu_features->kind = kind;
 326
 327 #if HAVE_TUNABLES
 328   TUNABLE_GET (hwcaps, tunable_val_t *, TUNABLE_CALLBACK (set_hwcaps));
 329   cpu_features->non_temporal_threshold
 330     = TUNABLE_GET (x86_non_temporal_threshold, long int, NULL);
 331   cpu_features->data_cache_size
 332     = TUNABLE_GET (x86_data_cache_size, long int, NULL);
 333   cpu_features->shared_cache_size
 334     = TUNABLE_GET (x86_shared_cache_size, long int, NULL);
 335 #endif
 336
 337   /* Reuse dl_platform, dl_hwcap and dl_hwcap_mask for x86.  */
 338 #if !HAVE_TUNABLES && defined SHARED
 339   /* The glibc.tune.hwcap_mask tunable is initialized already, so no need to do
 340      this.  */
 341   GLRO(dl_hwcap_mask) = HWCAP_IMPORTANT;
 342 #endif
 343
 344 #ifdef __x86_64__
 345   GLRO(dl_hwcap) = HWCAP_X86_64;
 346   if (cpu_features->kind == arch_kind_intel)
 347     {
 348       const char *platform = NULL;
 349
 350       if (CPU_FEATURES_ARCH_P (cpu_features, AVX512F_Usable)
 351           && CPU_FEATURES_CPU_P (cpu_features, AVX512CD))
 352         {
 353           if (CPU_FEATURES_CPU_P (cpu_features, AVX512ER))
 354             {
 355               if (CPU_FEATURES_CPU_P (cpu_features, AVX512PF))
 356                 platform = "xeon_phi";
 357             }
 358           else
 359             {
 360               if (CPU_FEATURES_CPU_P (cpu_features, AVX512BW)
 361                   && CPU_FEATURES_CPU_P (cpu_features, AVX512DQ)
 362                   && CPU_FEATURES_CPU_P (cpu_features, AVX512VL))
 363                 GLRO(dl_hwcap) |= HWCAP_X86_AVX512_1;
 364             }
 365         }
 366
 367       if (platform == NULL
 368           && CPU_FEATURES_ARCH_P (cpu_features, AVX2_Usable)
 369           && CPU_FEATURES_ARCH_P (cpu_features, FMA_Usable)
 370           && CPU_FEATURES_CPU_P (cpu_features, BMI1)
 371           && CPU_FEATURES_CPU_P (cpu_features, BMI2)
 372           && CPU_FEATURES_CPU_P (cpu_features, LZCNT)
 373           && CPU_FEATURES_CPU_P (cpu_features, MOVBE)
 374           && CPU_FEATURES_CPU_P (cpu_features, POPCNT))
 375         platform = "haswell";
 376
 377       if (platform != NULL)
 378         GLRO(dl_platform) = platform;
 379     }
 380 #else
 381   GLRO(dl_hwcap) = 0;
 382   if (CPU_FEATURES_CPU_P (cpu_features, SSE2))
 383     GLRO(dl_hwcap) |= HWCAP_X86_SSE2;
 384
 385   if (CPU_FEATURES_ARCH_P (cpu_features, I686))
 386     GLRO(dl_platform) = "i686";
 387   else if (CPU_FEATURES_ARCH_P (cpu_features, I586))
 388     GLRO(dl_platform) = "i586";
 389 #endif
 390 }