sysdeps/x86/cpu-features.c

   1 /* Initialize CPU feature data.
   2    This file is part of the GNU C Library.
   3    Copyright (C) 2008-2017 Free Software Foundation, Inc.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <http://www.gnu.org/licenses/>.  */
  18
  19 #include <cpuid.h>
  20 #include <cpu-features.h>
  21
  22 static void
  23 get_common_indeces (struct cpu_features *cpu_features,
  24                     unsigned int *family, unsigned int *model,
  25                     unsigned int *extended_model, unsigned int *stepping)
  26 {
  27   if (family)
  28     {
  29       unsigned int eax;
  30       __cpuid (1, eax, cpu_features->cpuid[COMMON_CPUID_INDEX_1].ebx,
  31                cpu_features->cpuid[COMMON_CPUID_INDEX_1].ecx,
  32                cpu_features->cpuid[COMMON_CPUID_INDEX_1].edx);
  33       cpu_features->cpuid[COMMON_CPUID_INDEX_1].eax = eax;
  34       *family = (eax >> 8) & 0x0f;
  35       *model = (eax >> 4) & 0x0f;
  36       *extended_model = (eax >> 12) & 0xf0;
  37       *stepping = eax & 0x0f;
  38       if (*family == 0x0f)
  39         {
  40           *family += (eax >> 20) & 0xff;
  41           *model += *extended_model;
  42         }
  43     }
  44
  45   if (cpu_features->max_cpuid >= 7)
  46     __cpuid_count (7, 0,
  47                    cpu_features->cpuid[COMMON_CPUID_INDEX_7].eax,
  48                    cpu_features->cpuid[COMMON_CPUID_INDEX_7].ebx,
  49                    cpu_features->cpuid[COMMON_CPUID_INDEX_7].ecx,
  50                    cpu_features->cpuid[COMMON_CPUID_INDEX_7].edx);
  51
  52   /* Can we call xgetbv?  */
  53   if (CPU_FEATURES_CPU_P (cpu_features, OSXSAVE))
  54     {
  55       unsigned int xcrlow;
  56       unsigned int xcrhigh;
  57       asm ("xgetbv" : "=a" (xcrlow), "=d" (xcrhigh) : "c" (0));
  58       /* Is YMM and XMM state usable?  */
  59       if ((xcrlow & (bit_YMM_state | bit_XMM_state)) ==
  60           (bit_YMM_state | bit_XMM_state))
  61         {
  62           /* Determine if AVX is usable.  */
  63           if (CPU_FEATURES_CPU_P (cpu_features, AVX))
  64             {
  65               cpu_features->feature[index_arch_AVX_Usable]
  66                 |= bit_arch_AVX_Usable;
  67               /* The following features depend on AVX being usable.  */
  68               /* Determine if AVX2 is usable.  */
  69               if (CPU_FEATURES_CPU_P (cpu_features, AVX2))
  70                 cpu_features->feature[index_arch_AVX2_Usable]
  71                   |= bit_arch_AVX2_Usable;
  72               /* Determine if FMA is usable.  */
  73               if (CPU_FEATURES_CPU_P (cpu_features, FMA))
  74                 cpu_features->feature[index_arch_FMA_Usable]
  75                   |= bit_arch_FMA_Usable;
  76             }
  77
  78           /* Check if OPMASK state, upper 256-bit of ZMM0-ZMM15 and
  79              ZMM16-ZMM31 state are enabled.  */
  80           if ((xcrlow & (bit_Opmask_state | bit_ZMM0_15_state
  81                          | bit_ZMM16_31_state)) ==
  82               (bit_Opmask_state | bit_ZMM0_15_state | bit_ZMM16_31_state))
  83             {
  84               /* Determine if AVX512F is usable.  */
  85               if (CPU_FEATURES_CPU_P (cpu_features, AVX512F))
  86                 {
  87                   cpu_features->feature[index_arch_AVX512F_Usable]
  88                     |= bit_arch_AVX512F_Usable;
  89                   /* Determine if AVX512DQ is usable.  */
  90                   if (CPU_FEATURES_CPU_P (cpu_features, AVX512DQ))
  91                     cpu_features->feature[index_arch_AVX512DQ_Usable]
  92                       |= bit_arch_AVX512DQ_Usable;
  93                 }
  94             }
  95         }
  96     }
  97 }
  98
  99 static inline void
 100 init_cpu_features (struct cpu_features *cpu_features)
 101 {
 102   unsigned int ebx, ecx, edx;
 103   unsigned int family = 0;
 104   unsigned int model = 0;
 105   enum cpu_features_kind kind;
 106
 107 #if !HAS_CPUID
 108   if (__get_cpuid_max (0, 0) == 0)
 109     {
 110       kind = arch_kind_other;
 111       goto no_cpuid;
 112     }
 113 #endif
 114
 115   __cpuid (0, cpu_features->max_cpuid, ebx, ecx, edx);
 116
 117   /* This spells out "GenuineIntel".  */
 118   if (ebx == 0x756e6547 && ecx == 0x6c65746e && edx == 0x49656e69)
 119     {
 120       unsigned int extended_model, stepping;
 121
 122       kind = arch_kind_intel;
 123
 124       get_common_indeces (cpu_features, &family, &model, &extended_model,
 125                           &stepping);
 126
 127       if (family == 0x06)
 128         {
 129           model += extended_model;
 130           switch (model)
 131             {
 132             case 0x1c:
 133             case 0x26:
 134               /* BSF is slow on Atom.  */
 135               cpu_features->feature[index_arch_Slow_BSF]
 136                 |= bit_arch_Slow_BSF;
 137               break;
 138
 139             case 0x57:
 140               /* Knights Landing.  Enable Silvermont optimizations.  */
 141
 142             case 0x5c:
 143             case 0x5f:
 144               /* Unaligned load versions are faster than SSSE3
 145                  on Goldmont.  */
 146
 147             case 0x4c:
 148               /* Airmont is a die shrink of Silvermont.  */
 149
 150             case 0x37:
 151             case 0x4a:
 152             case 0x4d:
 153             case 0x5a:
 154             case 0x5d:
 155               /* Unaligned load versions are faster than SSSE3
 156                  on Silvermont.  */
 157 #if index_arch_Fast_Unaligned_Load != index_arch_Prefer_PMINUB_for_stringop
 158 # error index_arch_Fast_Unaligned_Load != index_arch_Prefer_PMINUB_for_stringop
 159 #endif
 160 #if index_arch_Fast_Unaligned_Load != index_arch_Slow_SSE4_2
 161 # error index_arch_Fast_Unaligned_Load != index_arch_Slow_SSE4_2
 162 #endif
 163 #if index_arch_Fast_Unaligned_Load != index_arch_Fast_Unaligned_Copy
 164 # error index_arch_Fast_Unaligned_Load != index_arch_Fast_Unaligned_Copy
 165 #endif
 166               cpu_features->feature[index_arch_Fast_Unaligned_Load]
 167                 |= (bit_arch_Fast_Unaligned_Load
 168                     | bit_arch_Fast_Unaligned_Copy
 169                     | bit_arch_Prefer_PMINUB_for_stringop
 170                     | bit_arch_Slow_SSE4_2);
 171               break;
 172
 173             default:
 174               /* Unknown family 0x06 processors.  Assuming this is one
 175                  of Core i3/i5/i7 processors if AVX is available.  */
 176               if (!CPU_FEATURES_CPU_P (cpu_features, AVX))
 177                 break;
 178
 179             case 0x1a:
 180             case 0x1e:
 181             case 0x1f:
 182             case 0x25:
 183             case 0x2c:
 184             case 0x2e:
 185             case 0x2f:
 186               /* Rep string instructions, unaligned load, unaligned copy,
 187                  and pminub are fast on Intel Core i3, i5 and i7.  */
 188 #if index_arch_Fast_Rep_String != index_arch_Fast_Unaligned_Load
 189 # error index_arch_Fast_Rep_String != index_arch_Fast_Unaligned_Load
 190 #endif
 191 #if index_arch_Fast_Rep_String != index_arch_Prefer_PMINUB_for_stringop
 192 # error index_arch_Fast_Rep_String != index_arch_Prefer_PMINUB_for_stringop
 193 #endif
 194 #if index_arch_Fast_Rep_String != index_arch_Fast_Unaligned_Copy
 195 # error index_arch_Fast_Rep_String != index_arch_Fast_Unaligned_Copy
 196 #endif
 197               cpu_features->feature[index_arch_Fast_Rep_String]
 198                 |= (bit_arch_Fast_Rep_String
 199                     | bit_arch_Fast_Unaligned_Load
 200                     | bit_arch_Fast_Unaligned_Copy
 201                     | bit_arch_Prefer_PMINUB_for_stringop);
 202               break;
 203
 204             case 0x3f:
 205               /* Xeon E7 v3 with stepping >= 4 has working TSX.  */
 206               if (stepping >= 4)
 207                 break;
 208             case 0x3c:
 209             case 0x45:
 210             case 0x46:
 211               /* Disable Intel TSX on Haswell processors (except Xeon E7 v3
 212                  with stepping >= 4) to avoid TSX on kernels that weren't
 213                  updated with the latest microcode package (which disables
 214                  broken feature by default).  */
 215               cpu_features->cpuid[index_cpu_RTM].reg_RTM &= ~bit_cpu_RTM;
 216               break;
 217             }
 218         }
 219
 220       /* Unaligned load with 256-bit AVX registers are faster on
 221          Intel processors with AVX2.  */
 222       if (CPU_FEATURES_ARCH_P (cpu_features, AVX2_Usable))
 223         cpu_features->feature[index_arch_AVX_Fast_Unaligned_Load]
 224           |= bit_arch_AVX_Fast_Unaligned_Load;
 225
 226       /* Since AVX512ER is unique to Xeon Phi, set Prefer_No_VZEROUPPER
 227          if AVX512ER is available.  Don't use AVX512 to avoid lower CPU
 228          frequency if AVX512ER isn't available.  */
 229       if (CPU_FEATURES_CPU_P (cpu_features, AVX512ER))
 230         cpu_features->feature[index_arch_Prefer_No_VZEROUPPER]
 231           |= bit_arch_Prefer_No_VZEROUPPER;
 232       else
 233         cpu_features->feature[index_arch_Prefer_No_AVX512]
 234           |= bit_arch_Prefer_No_AVX512;
 235
 236       /* To avoid SSE transition penalty, use _dl_runtime_resolve_slow.
 237          If XGETBV suports ECX == 1, use _dl_runtime_resolve_opt.  */
 238       cpu_features->feature[index_arch_Use_dl_runtime_resolve_slow]
 239         |= bit_arch_Use_dl_runtime_resolve_slow;
 240       if (cpu_features->max_cpuid >= 0xd)
 241         {
 242           unsigned int eax;
 243
 244           __cpuid_count (0xd, 1, eax, ebx, ecx, edx);
 245           if ((eax & (1 << 2)) != 0)
 246             cpu_features->feature[index_arch_Use_dl_runtime_resolve_opt]
 247               |= bit_arch_Use_dl_runtime_resolve_opt;
 248         }
 249     }
 250   /* This spells out "AuthenticAMD".  */
 251   else if (ebx == 0x68747541 && ecx == 0x444d4163 && edx == 0x69746e65)
 252     {
 253       unsigned int extended_model, stepping;
 254
 255       kind = arch_kind_amd;
 256
 257       get_common_indeces (cpu_features, &family, &model, &extended_model,
 258                           &stepping);
 259
 260       ecx = cpu_features->cpuid[COMMON_CPUID_INDEX_1].ecx;
 261
 262       unsigned int eax;
 263       __cpuid (0x80000000, eax, ebx, ecx, edx);
 264       if (eax >= 0x80000001)
 265         __cpuid (0x80000001,
 266                  cpu_features->cpuid[COMMON_CPUID_INDEX_80000001].eax,
 267                  cpu_features->cpuid[COMMON_CPUID_INDEX_80000001].ebx,
 268                  cpu_features->cpuid[COMMON_CPUID_INDEX_80000001].ecx,
 269                  cpu_features->cpuid[COMMON_CPUID_INDEX_80000001].edx);
 270
 271       if (HAS_ARCH_FEATURE (AVX_Usable))
 272         {
 273           /* Since the FMA4 bit is in COMMON_CPUID_INDEX_80000001 and
 274              FMA4 requires AVX, determine if FMA4 is usable here.  */
 275           if (CPU_FEATURES_CPU_P (cpu_features, FMA4))
 276             cpu_features->feature[index_arch_FMA4_Usable]
 277               |= bit_arch_FMA4_Usable;
 278         }
 279
 280       if (family == 0x15)
 281         {
 282 #if index_arch_Fast_Unaligned_Load != index_arch_Fast_Copy_Backward
 283 # error index_arch_Fast_Unaligned_Load != index_arch_Fast_Copy_Backward
 284 #endif
 285           /* "Excavator"   */
 286           if (model >= 0x60 && model <= 0x7f)
 287             cpu_features->feature[index_arch_Fast_Unaligned_Load]
 288               |= (bit_arch_Fast_Unaligned_Load
 289                   | bit_arch_Fast_Copy_Backward);
 290         }
 291     }
 292   else
 293     {
 294       kind = arch_kind_other;
 295       get_common_indeces (cpu_features, NULL, NULL, NULL, NULL);
 296     }
 297
 298   /* Support i586 if CX8 is available.  */
 299   if (CPU_FEATURES_CPU_P (cpu_features, CX8))
 300     cpu_features->feature[index_arch_I586] |= bit_arch_I586;
 301
 302   /* Support i686 if CMOV is available.  */
 303   if (CPU_FEATURES_CPU_P (cpu_features, CMOV))
 304     cpu_features->feature[index_arch_I686] |= bit_arch_I686;
 305
 306 #if !HAS_CPUID
 307 no_cpuid:
 308 #endif
 309
 310   cpu_features->family = family;
 311   cpu_features->model = model;
 312   cpu_features->kind = kind;
 313 }