sysdeps/x86/cpu-features.c

   1 /* Initialize CPU feature data.
   2    This file is part of the GNU C Library.
   3    Copyright (C) 2008-2017 Free Software Foundation, Inc.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <http://www.gnu.org/licenses/>.  */
  18
  19 #include <cpuid.h>
  20 #include <cpu-features.h>
  21 #include <dl-hwcap.h>
  22
  23 static void
  24 get_common_indeces (struct cpu_features *cpu_features,
  25                     unsigned int *family, unsigned int *model,
  26                     unsigned int *extended_model, unsigned int *stepping)
  27 {
  28   if (family)
  29     {
  30       unsigned int eax;
  31       __cpuid (1, eax, cpu_features->cpuid[COMMON_CPUID_INDEX_1].ebx,
  32                cpu_features->cpuid[COMMON_CPUID_INDEX_1].ecx,
  33                cpu_features->cpuid[COMMON_CPUID_INDEX_1].edx);
  34       cpu_features->cpuid[COMMON_CPUID_INDEX_1].eax = eax;
  35       *family = (eax >> 8) & 0x0f;
  36       *model = (eax >> 4) & 0x0f;
  37       *extended_model = (eax >> 12) & 0xf0;
  38       *stepping = eax & 0x0f;
  39       if (*family == 0x0f)
  40         {
  41           *family += (eax >> 20) & 0xff;
  42           *model += *extended_model;
  43         }
  44     }
  45
  46   if (cpu_features->max_cpuid >= 7)
  47     __cpuid_count (7, 0,
  48                    cpu_features->cpuid[COMMON_CPUID_INDEX_7].eax,
  49                    cpu_features->cpuid[COMMON_CPUID_INDEX_7].ebx,
  50                    cpu_features->cpuid[COMMON_CPUID_INDEX_7].ecx,
  51                    cpu_features->cpuid[COMMON_CPUID_INDEX_7].edx);
  52
  53   /* Can we call xgetbv?  */
  54   if (CPU_FEATURES_CPU_P (cpu_features, OSXSAVE))
  55     {
  56       unsigned int xcrlow;
  57       unsigned int xcrhigh;
  58       asm ("xgetbv" : "=a" (xcrlow), "=d" (xcrhigh) : "c" (0));
  59       /* Is YMM and XMM state usable?  */
  60       if ((xcrlow & (bit_YMM_state | bit_XMM_state)) ==
  61           (bit_YMM_state | bit_XMM_state))
  62         {
  63           /* Determine if AVX is usable.  */
  64           if (CPU_FEATURES_CPU_P (cpu_features, AVX))
  65             {
  66               cpu_features->feature[index_arch_AVX_Usable]
  67                 |= bit_arch_AVX_Usable;
  68               /* The following features depend on AVX being usable.  */
  69               /* Determine if AVX2 is usable.  */
  70               if (CPU_FEATURES_CPU_P (cpu_features, AVX2))
  71                 cpu_features->feature[index_arch_AVX2_Usable]
  72                   |= bit_arch_AVX2_Usable;
  73               /* Determine if FMA is usable.  */
  74               if (CPU_FEATURES_CPU_P (cpu_features, FMA))
  75                 cpu_features->feature[index_arch_FMA_Usable]
  76                   |= bit_arch_FMA_Usable;
  77             }
  78
  79           /* Check if OPMASK state, upper 256-bit of ZMM0-ZMM15 and
  80              ZMM16-ZMM31 state are enabled.  */
  81           if ((xcrlow & (bit_Opmask_state | bit_ZMM0_15_state
  82                          | bit_ZMM16_31_state)) ==
  83               (bit_Opmask_state | bit_ZMM0_15_state | bit_ZMM16_31_state))
  84             {
  85               /* Determine if AVX512F is usable.  */
  86               if (CPU_FEATURES_CPU_P (cpu_features, AVX512F))
  87                 {
  88                   cpu_features->feature[index_arch_AVX512F_Usable]
  89                     |= bit_arch_AVX512F_Usable;
  90                   /* Determine if AVX512DQ is usable.  */
  91                   if (CPU_FEATURES_CPU_P (cpu_features, AVX512DQ))
  92                     cpu_features->feature[index_arch_AVX512DQ_Usable]
  93                       |= bit_arch_AVX512DQ_Usable;
  94                 }
  95             }
  96         }
  97     }
  98 }
  99
 100 static inline void
 101 init_cpu_features (struct cpu_features *cpu_features)
 102 {
 103   unsigned int ebx, ecx, edx;
 104   unsigned int family = 0;
 105   unsigned int model = 0;
 106   enum cpu_features_kind kind;
 107
 108 #if !HAS_CPUID
 109   if (__get_cpuid_max (0, 0) == 0)
 110     {
 111       kind = arch_kind_other;
 112       goto no_cpuid;
 113     }
 114 #endif
 115
 116   __cpuid (0, cpu_features->max_cpuid, ebx, ecx, edx);
 117
 118   /* This spells out "GenuineIntel".  */
 119   if (ebx == 0x756e6547 && ecx == 0x6c65746e && edx == 0x49656e69)
 120     {
 121       unsigned int extended_model, stepping;
 122
 123       kind = arch_kind_intel;
 124
 125       get_common_indeces (cpu_features, &family, &model, &extended_model,
 126                           &stepping);
 127
 128       if (family == 0x06)
 129         {
 130           model += extended_model;
 131           switch (model)
 132             {
 133             case 0x1c:
 134             case 0x26:
 135               /* BSF is slow on Atom.  */
 136               cpu_features->feature[index_arch_Slow_BSF]
 137                 |= bit_arch_Slow_BSF;
 138               break;
 139
 140             case 0x57:
 141               /* Knights Landing.  Enable Silvermont optimizations.  */
 142
 143             case 0x5c:
 144             case 0x5f:
 145               /* Unaligned load versions are faster than SSSE3
 146                  on Goldmont.  */
 147
 148             case 0x4c:
 149               /* Airmont is a die shrink of Silvermont.  */
 150
 151             case 0x37:
 152             case 0x4a:
 153             case 0x4d:
 154             case 0x5a:
 155             case 0x5d:
 156               /* Unaligned load versions are faster than SSSE3
 157                  on Silvermont.  */
 158 #if index_arch_Fast_Unaligned_Load != index_arch_Prefer_PMINUB_for_stringop
 159 # error index_arch_Fast_Unaligned_Load != index_arch_Prefer_PMINUB_for_stringop
 160 #endif
 161 #if index_arch_Fast_Unaligned_Load != index_arch_Slow_SSE4_2
 162 # error index_arch_Fast_Unaligned_Load != index_arch_Slow_SSE4_2
 163 #endif
 164 #if index_arch_Fast_Unaligned_Load != index_arch_Fast_Unaligned_Copy
 165 # error index_arch_Fast_Unaligned_Load != index_arch_Fast_Unaligned_Copy
 166 #endif
 167               cpu_features->feature[index_arch_Fast_Unaligned_Load]
 168                 |= (bit_arch_Fast_Unaligned_Load
 169                     | bit_arch_Fast_Unaligned_Copy
 170                     | bit_arch_Prefer_PMINUB_for_stringop
 171                     | bit_arch_Slow_SSE4_2);
 172               break;
 173
 174             default:
 175               /* Unknown family 0x06 processors.  Assuming this is one
 176                  of Core i3/i5/i7 processors if AVX is available.  */
 177               if (!CPU_FEATURES_CPU_P (cpu_features, AVX))
 178                 break;
 179
 180             case 0x1a:
 181             case 0x1e:
 182             case 0x1f:
 183             case 0x25:
 184             case 0x2c:
 185             case 0x2e:
 186             case 0x2f:
 187               /* Rep string instructions, unaligned load, unaligned copy,
 188                  and pminub are fast on Intel Core i3, i5 and i7.  */
 189 #if index_arch_Fast_Rep_String != index_arch_Fast_Unaligned_Load
 190 # error index_arch_Fast_Rep_String != index_arch_Fast_Unaligned_Load
 191 #endif
 192 #if index_arch_Fast_Rep_String != index_arch_Prefer_PMINUB_for_stringop
 193 # error index_arch_Fast_Rep_String != index_arch_Prefer_PMINUB_for_stringop
 194 #endif
 195 #if index_arch_Fast_Rep_String != index_arch_Fast_Unaligned_Copy
 196 # error index_arch_Fast_Rep_String != index_arch_Fast_Unaligned_Copy
 197 #endif
 198               cpu_features->feature[index_arch_Fast_Rep_String]
 199                 |= (bit_arch_Fast_Rep_String
 200                     | bit_arch_Fast_Unaligned_Load
 201                     | bit_arch_Fast_Unaligned_Copy
 202                     | bit_arch_Prefer_PMINUB_for_stringop);
 203               break;
 204
 205             case 0x3f:
 206               /* Xeon E7 v3 with stepping >= 4 has working TSX.  */
 207               if (stepping >= 4)
 208                 break;
 209             case 0x3c:
 210             case 0x45:
 211             case 0x46:
 212               /* Disable Intel TSX on Haswell processors (except Xeon E7 v3
 213                  with stepping >= 4) to avoid TSX on kernels that weren't
 214                  updated with the latest microcode package (which disables
 215                  broken feature by default).  */
 216               cpu_features->cpuid[index_cpu_RTM].reg_RTM &= ~bit_cpu_RTM;
 217               break;
 218             }
 219         }
 220
 221       /* Unaligned load with 256-bit AVX registers are faster on
 222          Intel processors with AVX2.  */
 223       if (CPU_FEATURES_ARCH_P (cpu_features, AVX2_Usable))
 224         cpu_features->feature[index_arch_AVX_Fast_Unaligned_Load]
 225           |= bit_arch_AVX_Fast_Unaligned_Load;
 226
 227       /* Since AVX512ER is unique to Xeon Phi, set Prefer_No_VZEROUPPER
 228          if AVX512ER is available.  Don't use AVX512 to avoid lower CPU
 229          frequency if AVX512ER isn't available.  */
 230       if (CPU_FEATURES_CPU_P (cpu_features, AVX512ER))
 231         cpu_features->feature[index_arch_Prefer_No_VZEROUPPER]
 232           |= bit_arch_Prefer_No_VZEROUPPER;
 233       else
 234         cpu_features->feature[index_arch_Prefer_No_AVX512]
 235           |= bit_arch_Prefer_No_AVX512;
 236
 237       /* To avoid SSE transition penalty, use _dl_runtime_resolve_slow.
 238          If XGETBV suports ECX == 1, use _dl_runtime_resolve_opt.  */
 239       cpu_features->feature[index_arch_Use_dl_runtime_resolve_slow]
 240         |= bit_arch_Use_dl_runtime_resolve_slow;
 241       if (cpu_features->max_cpuid >= 0xd)
 242         {
 243           unsigned int eax;
 244
 245           __cpuid_count (0xd, 1, eax, ebx, ecx, edx);
 246           if ((eax & (1 << 2)) != 0)
 247             cpu_features->feature[index_arch_Use_dl_runtime_resolve_opt]
 248               |= bit_arch_Use_dl_runtime_resolve_opt;
 249         }
 250     }
 251   /* This spells out "AuthenticAMD".  */
 252   else if (ebx == 0x68747541 && ecx == 0x444d4163 && edx == 0x69746e65)
 253     {
 254       unsigned int extended_model, stepping;
 255
 256       kind = arch_kind_amd;
 257
 258       get_common_indeces (cpu_features, &family, &model, &extended_model,
 259                           &stepping);
 260
 261       ecx = cpu_features->cpuid[COMMON_CPUID_INDEX_1].ecx;
 262
 263       unsigned int eax;
 264       __cpuid (0x80000000, eax, ebx, ecx, edx);
 265       if (eax >= 0x80000001)
 266         __cpuid (0x80000001,
 267                  cpu_features->cpuid[COMMON_CPUID_INDEX_80000001].eax,
 268                  cpu_features->cpuid[COMMON_CPUID_INDEX_80000001].ebx,
 269                  cpu_features->cpuid[COMMON_CPUID_INDEX_80000001].ecx,
 270                  cpu_features->cpuid[COMMON_CPUID_INDEX_80000001].edx);
 271
 272       if (HAS_ARCH_FEATURE (AVX_Usable))
 273         {
 274           /* Since the FMA4 bit is in COMMON_CPUID_INDEX_80000001 and
 275              FMA4 requires AVX, determine if FMA4 is usable here.  */
 276           if (CPU_FEATURES_CPU_P (cpu_features, FMA4))
 277             cpu_features->feature[index_arch_FMA4_Usable]
 278               |= bit_arch_FMA4_Usable;
 279         }
 280
 281       if (family == 0x15)
 282         {
 283 #if index_arch_Fast_Unaligned_Load != index_arch_Fast_Copy_Backward
 284 # error index_arch_Fast_Unaligned_Load != index_arch_Fast_Copy_Backward
 285 #endif
 286           /* "Excavator"   */
 287           if (model >= 0x60 && model <= 0x7f)
 288             cpu_features->feature[index_arch_Fast_Unaligned_Load]
 289               |= (bit_arch_Fast_Unaligned_Load
 290                   | bit_arch_Fast_Copy_Backward);
 291         }
 292     }
 293   else
 294     {
 295       kind = arch_kind_other;
 296       get_common_indeces (cpu_features, NULL, NULL, NULL, NULL);
 297     }
 298
 299   /* Support i586 if CX8 is available.  */
 300   if (CPU_FEATURES_CPU_P (cpu_features, CX8))
 301     cpu_features->feature[index_arch_I586] |= bit_arch_I586;
 302
 303   /* Support i686 if CMOV is available.  */
 304   if (CPU_FEATURES_CPU_P (cpu_features, CMOV))
 305     cpu_features->feature[index_arch_I686] |= bit_arch_I686;
 306
 307 #if !HAS_CPUID
 308 no_cpuid:
 309 #endif
 310
 311   cpu_features->family = family;
 312   cpu_features->model = model;
 313   cpu_features->kind = kind;
 314
 315   /* Reuse dl_platform, dl_hwcap and dl_hwcap_mask for x86.  */
 316   GLRO(dl_platform) = NULL;
 317   GLRO(dl_hwcap) = 0;
 318 #if !HAVE_TUNABLES && defined SHARED
 319   /* The glibc.tune.hwcap_mask tunable is initialized already, so no need to do
 320      this.  */
 321   GLRO(dl_hwcap_mask) = HWCAP_IMPORTANT;
 322 #endif
 323
 324 #ifdef __x86_64__
 325   if (cpu_features->kind == arch_kind_intel)
 326     {
 327       if (CPU_FEATURES_ARCH_P (cpu_features, AVX512F_Usable)
 328           && CPU_FEATURES_CPU_P (cpu_features, AVX512CD))
 329         {
 330           if (CPU_FEATURES_CPU_P (cpu_features, AVX512ER))
 331             {
 332               if (CPU_FEATURES_CPU_P (cpu_features, AVX512PF))
 333                 GLRO(dl_platform) = "xeon_phi";
 334             }
 335           else
 336             {
 337               if (CPU_FEATURES_CPU_P (cpu_features, AVX512BW)
 338                   && CPU_FEATURES_CPU_P (cpu_features, AVX512DQ)
 339                   && CPU_FEATURES_CPU_P (cpu_features, AVX512VL))
 340                 GLRO(dl_hwcap) |= HWCAP_X86_AVX512_1;
 341             }
 342         }
 343
 344       if (GLRO(dl_platform) == NULL
 345           && CPU_FEATURES_ARCH_P (cpu_features, AVX2_Usable)
 346           && CPU_FEATURES_ARCH_P (cpu_features, FMA_Usable)
 347           && CPU_FEATURES_CPU_P (cpu_features, BMI1)
 348           && CPU_FEATURES_CPU_P (cpu_features, BMI2)
 349           && CPU_FEATURES_CPU_P (cpu_features, LZCNT)
 350           && CPU_FEATURES_CPU_P (cpu_features, MOVBE)
 351           && CPU_FEATURES_CPU_P (cpu_features, POPCNT))
 352         GLRO(dl_platform) = "haswell";
 353     }
 354 #else
 355   if (CPU_FEATURES_CPU_P (cpu_features, SSE2))
 356     GLRO(dl_hwcap) |= HWCAP_X86_SSE2;
 357
 358   if (CPU_FEATURES_ARCH_P (cpu_features, I686))
 359     GLRO(dl_platform) = "i686";
 360   else if (CPU_FEATURES_ARCH_P (cpu_features, I586))
 361     GLRO(dl_platform) = "i586";
 362 #endif
 363 }