sysdeps/x86/cpu-features.c

   1 /* Initialize CPU feature data.
   2    This file is part of the GNU C Library.
   3    Copyright (C) 2008-2023 Free Software Foundation, Inc.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <https://www.gnu.org/licenses/>.  */
  18
  19 #include <dl-hwcap.h>
  20 #include <libc-pointer-arith.h>
  21 #include <get-isa-level.h>
  22 #include <cacheinfo.h>
  23 #include <dl-cacheinfo.h>
  24 #include <dl-minsigstacksize.h>
  25 #include <dl-hwcap2.h>
  26
  27 extern void TUNABLE_CALLBACK (set_hwcaps) (tunable_val_t *)
  28   attribute_hidden;
  29
  30 #ifdef __LP64__
  31 static void
  32 TUNABLE_CALLBACK (set_prefer_map_32bit_exec) (tunable_val_t *valp)
  33 {
  34   if (valp->numval)
  35     GLRO(dl_x86_cpu_features).preferred[index_arch_Prefer_MAP_32BIT_EXEC]
  36       |= bit_arch_Prefer_MAP_32BIT_EXEC;
  37 }
  38 #endif
  39
  40 #if CET_ENABLED
  41 extern void TUNABLE_CALLBACK (set_x86_ibt) (tunable_val_t *)
  42   attribute_hidden;
  43 extern void TUNABLE_CALLBACK (set_x86_shstk) (tunable_val_t *)
  44   attribute_hidden;
  45
  46 # include <dl-cet.h>
  47 #endif
  48
  49 static void
  50 update_active (struct cpu_features *cpu_features)
  51 {
  52   /* Copy the cpuid bits to active bits for CPU featuress whose usability
  53      in user space can be detected without additional OS support.  */
  54   CPU_FEATURE_SET_ACTIVE (cpu_features, SSE3);
  55   CPU_FEATURE_SET_ACTIVE (cpu_features, PCLMULQDQ);
  56   CPU_FEATURE_SET_ACTIVE (cpu_features, SSSE3);
  57   CPU_FEATURE_SET_ACTIVE (cpu_features, CMPXCHG16B);
  58   CPU_FEATURE_SET_ACTIVE (cpu_features, SSE4_1);
  59   CPU_FEATURE_SET_ACTIVE (cpu_features, SSE4_2);
  60   CPU_FEATURE_SET_ACTIVE (cpu_features, MOVBE);
  61   CPU_FEATURE_SET_ACTIVE (cpu_features, POPCNT);
  62   CPU_FEATURE_SET_ACTIVE (cpu_features, AES);
  63   CPU_FEATURE_SET_ACTIVE (cpu_features, OSXSAVE);
  64   CPU_FEATURE_SET_ACTIVE (cpu_features, TSC);
  65   CPU_FEATURE_SET_ACTIVE (cpu_features, CX8);
  66   CPU_FEATURE_SET_ACTIVE (cpu_features, CMOV);
  67   CPU_FEATURE_SET_ACTIVE (cpu_features, CLFSH);
  68   CPU_FEATURE_SET_ACTIVE (cpu_features, MMX);
  69   CPU_FEATURE_SET_ACTIVE (cpu_features, FXSR);
  70   CPU_FEATURE_SET_ACTIVE (cpu_features, SSE);
  71   CPU_FEATURE_SET_ACTIVE (cpu_features, SSE2);
  72   CPU_FEATURE_SET_ACTIVE (cpu_features, HTT);
  73   CPU_FEATURE_SET_ACTIVE (cpu_features, BMI1);
  74   CPU_FEATURE_SET_ACTIVE (cpu_features, HLE);
  75   CPU_FEATURE_SET_ACTIVE (cpu_features, BMI2);
  76   CPU_FEATURE_SET_ACTIVE (cpu_features, ERMS);
  77   CPU_FEATURE_SET_ACTIVE (cpu_features, RDSEED);
  78   CPU_FEATURE_SET_ACTIVE (cpu_features, ADX);
  79   CPU_FEATURE_SET_ACTIVE (cpu_features, CLFLUSHOPT);
  80   CPU_FEATURE_SET_ACTIVE (cpu_features, CLWB);
  81   CPU_FEATURE_SET_ACTIVE (cpu_features, SHA);
  82   CPU_FEATURE_SET_ACTIVE (cpu_features, PREFETCHWT1);
  83   CPU_FEATURE_SET_ACTIVE (cpu_features, OSPKE);
  84   CPU_FEATURE_SET_ACTIVE (cpu_features, WAITPKG);
  85   CPU_FEATURE_SET_ACTIVE (cpu_features, GFNI);
  86   CPU_FEATURE_SET_ACTIVE (cpu_features, RDPID);
  87   CPU_FEATURE_SET_ACTIVE (cpu_features, RDRAND);
  88   CPU_FEATURE_SET_ACTIVE (cpu_features, CLDEMOTE);
  89   CPU_FEATURE_SET_ACTIVE (cpu_features, MOVDIRI);
  90   CPU_FEATURE_SET_ACTIVE (cpu_features, MOVDIR64B);
  91   CPU_FEATURE_SET_ACTIVE (cpu_features, FSRM);
  92   CPU_FEATURE_SET_ACTIVE (cpu_features, RTM_ALWAYS_ABORT);
  93   CPU_FEATURE_SET_ACTIVE (cpu_features, SERIALIZE);
  94   CPU_FEATURE_SET_ACTIVE (cpu_features, TSXLDTRK);
  95   CPU_FEATURE_SET_ACTIVE (cpu_features, LAHF64_SAHF64);
  96   CPU_FEATURE_SET_ACTIVE (cpu_features, LZCNT);
  97   CPU_FEATURE_SET_ACTIVE (cpu_features, SSE4A);
  98   CPU_FEATURE_SET_ACTIVE (cpu_features, PREFETCHW);
  99   CPU_FEATURE_SET_ACTIVE (cpu_features, TBM);
 100   CPU_FEATURE_SET_ACTIVE (cpu_features, RDTSCP);
 101   CPU_FEATURE_SET_ACTIVE (cpu_features, WBNOINVD);
 102   CPU_FEATURE_SET_ACTIVE (cpu_features, RAO_INT);
 103   CPU_FEATURE_SET_ACTIVE (cpu_features, CMPCCXADD);
 104   CPU_FEATURE_SET_ACTIVE (cpu_features, FZLRM);
 105   CPU_FEATURE_SET_ACTIVE (cpu_features, FSRS);
 106   CPU_FEATURE_SET_ACTIVE (cpu_features, FSRCS);
 107   CPU_FEATURE_SET_ACTIVE (cpu_features, PREFETCHI);
 108   CPU_FEATURE_SET_ACTIVE (cpu_features, PTWRITE);
 109
 110   if (!CPU_FEATURES_CPU_P (cpu_features, RTM_ALWAYS_ABORT))
 111     CPU_FEATURE_SET_ACTIVE (cpu_features, RTM);
 112
 113 #if CET_ENABLED
 114   CPU_FEATURE_SET_ACTIVE (cpu_features, IBT);
 115   CPU_FEATURE_SET_ACTIVE (cpu_features, SHSTK);
 116 #endif
 117
 118   enum
 119   {
 120     os_xmm = 1,
 121     os_ymm = 2,
 122     os_zmm = 4
 123   } os_vector_size = os_xmm;
 124   /* Can we call xgetbv?  */
 125   if (CPU_FEATURES_CPU_P (cpu_features, OSXSAVE))
 126     {
 127       unsigned int xcrlow;
 128       unsigned int xcrhigh;
 129       CPU_FEATURE_SET_ACTIVE (cpu_features, AVX10);
 130       asm ("xgetbv" : "=a" (xcrlow), "=d" (xcrhigh) : "c" (0));
 131       /* Is YMM and XMM state usable?  */
 132       if ((xcrlow & (bit_YMM_state | bit_XMM_state))
 133           == (bit_YMM_state | bit_XMM_state))
 134         {
 135           /* Determine if AVX is usable.  */
 136           if (CPU_FEATURES_CPU_P (cpu_features, AVX))
 137             {
 138               os_vector_size |= os_ymm;
 139               CPU_FEATURE_SET (cpu_features, AVX);
 140               /* The following features depend on AVX being usable.  */
 141               /* Determine if AVX2 is usable.  */
 142               if (CPU_FEATURES_CPU_P (cpu_features, AVX2))
 143                 {
 144                   CPU_FEATURE_SET (cpu_features, AVX2);
 145
 146                   /* Unaligned load with 256-bit AVX registers are faster
 147                      on Intel/AMD processors with AVX2.  */
 148                   cpu_features->preferred[index_arch_AVX_Fast_Unaligned_Load]
 149                     |= bit_arch_AVX_Fast_Unaligned_Load;
 150                 }
 151               /* Determine if AVX-IFMA is usable.  */
 152               CPU_FEATURE_SET_ACTIVE (cpu_features, AVX_IFMA);
 153               /* Determine if AVX-NE-CONVERT is usable.  */
 154               CPU_FEATURE_SET_ACTIVE (cpu_features, AVX_NE_CONVERT);
 155               /* Determine if AVX-VNNI is usable.  */
 156               CPU_FEATURE_SET_ACTIVE (cpu_features, AVX_VNNI);
 157               /* Determine if AVX-VNNI-INT8 is usable.  */
 158               CPU_FEATURE_SET_ACTIVE (cpu_features, AVX_VNNI_INT8);
 159               /* Determine if FMA is usable.  */
 160               CPU_FEATURE_SET_ACTIVE (cpu_features, FMA);
 161               /* Determine if VAES is usable.  */
 162               CPU_FEATURE_SET_ACTIVE (cpu_features, VAES);
 163               /* Determine if VPCLMULQDQ is usable.  */
 164               CPU_FEATURE_SET_ACTIVE (cpu_features, VPCLMULQDQ);
 165               /* Determine if XOP is usable.  */
 166               CPU_FEATURE_SET_ACTIVE (cpu_features, XOP);
 167               /* Determine if F16C is usable.  */
 168               CPU_FEATURE_SET_ACTIVE (cpu_features, F16C);
 169             }
 170
 171           /* Check if OPMASK state, upper 256-bit of ZMM0-ZMM15 and
 172              ZMM16-ZMM31 state are enabled.  */
 173           if ((xcrlow & (bit_Opmask_state | bit_ZMM0_15_state
 174                          | bit_ZMM16_31_state))
 175               == (bit_Opmask_state | bit_ZMM0_15_state | bit_ZMM16_31_state))
 176             {
 177               os_vector_size |= os_zmm;
 178               /* Determine if AVX512F is usable.  */
 179               if (CPU_FEATURES_CPU_P (cpu_features, AVX512F))
 180                 {
 181                   CPU_FEATURE_SET (cpu_features, AVX512F);
 182                   /* Determine if AVX512CD is usable.  */
 183                   CPU_FEATURE_SET_ACTIVE (cpu_features, AVX512CD);
 184                   /* Determine if AVX512ER is usable.  */
 185                   CPU_FEATURE_SET_ACTIVE (cpu_features, AVX512ER);
 186                   /* Determine if AVX512PF is usable.  */
 187                   CPU_FEATURE_SET_ACTIVE (cpu_features, AVX512PF);
 188                   /* Determine if AVX512VL is usable.  */
 189                   CPU_FEATURE_SET_ACTIVE (cpu_features, AVX512VL);
 190                   /* Determine if AVX512DQ is usable.  */
 191                   CPU_FEATURE_SET_ACTIVE (cpu_features, AVX512DQ);
 192                   /* Determine if AVX512BW is usable.  */
 193                   CPU_FEATURE_SET_ACTIVE (cpu_features, AVX512BW);
 194                   /* Determine if AVX512_4FMAPS is usable.  */
 195                   CPU_FEATURE_SET_ACTIVE (cpu_features, AVX512_4FMAPS);
 196                   /* Determine if AVX512_4VNNIW is usable.  */
 197                   CPU_FEATURE_SET_ACTIVE (cpu_features, AVX512_4VNNIW);
 198                   /* Determine if AVX512_BITALG is usable.  */
 199                   CPU_FEATURE_SET_ACTIVE (cpu_features, AVX512_BITALG);
 200                   /* Determine if AVX512_IFMA is usable.  */
 201                   CPU_FEATURE_SET_ACTIVE (cpu_features, AVX512_IFMA);
 202                   /* Determine if AVX512_VBMI is usable.  */
 203                   CPU_FEATURE_SET_ACTIVE (cpu_features, AVX512_VBMI);
 204                   /* Determine if AVX512_VBMI2 is usable.  */
 205                   CPU_FEATURE_SET_ACTIVE (cpu_features, AVX512_VBMI2);
 206                   /* Determine if is AVX512_VNNI usable.  */
 207                   CPU_FEATURE_SET_ACTIVE (cpu_features, AVX512_VNNI);
 208                   /* Determine if AVX512_VPOPCNTDQ is usable.  */
 209                   CPU_FEATURE_SET_ACTIVE (cpu_features,
 210                                           AVX512_VPOPCNTDQ);
 211                   /* Determine if AVX512_VP2INTERSECT is usable.  */
 212                   CPU_FEATURE_SET_ACTIVE (cpu_features,
 213                                           AVX512_VP2INTERSECT);
 214                   /* Determine if AVX512_BF16 is usable.  */
 215                   CPU_FEATURE_SET_ACTIVE (cpu_features, AVX512_BF16);
 216                   /* Determine if AVX512_FP16 is usable.  */
 217                   CPU_FEATURE_SET_ACTIVE (cpu_features, AVX512_FP16);
 218                 }
 219             }
 220         }
 221
 222       if (CPU_FEATURES_CPU_P (cpu_features, AVX10)
 223           && cpu_features->basic.max_cpuid >= 0x24)
 224         {
 225           __cpuid_count (
 226               0x24, 0, cpu_features->features[CPUID_INDEX_24_ECX_0].cpuid.eax,
 227               cpu_features->features[CPUID_INDEX_24_ECX_0].cpuid.ebx,
 228               cpu_features->features[CPUID_INDEX_24_ECX_0].cpuid.ecx,
 229               cpu_features->features[CPUID_INDEX_24_ECX_0].cpuid.edx);
 230           if (os_vector_size & os_xmm)
 231             CPU_FEATURE_SET_ACTIVE (cpu_features, AVX10_XMM);
 232           if (os_vector_size & os_ymm)
 233             CPU_FEATURE_SET_ACTIVE (cpu_features, AVX10_YMM);
 234           if (os_vector_size & os_zmm)
 235             CPU_FEATURE_SET_ACTIVE (cpu_features, AVX10_ZMM);
 236         }
 237
 238       /* Are XTILECFG and XTILEDATA states usable?  */
 239       if ((xcrlow & (bit_XTILECFG_state | bit_XTILEDATA_state))
 240           == (bit_XTILECFG_state | bit_XTILEDATA_state))
 241         {
 242           /* Determine if AMX_BF16 is usable.  */
 243           CPU_FEATURE_SET_ACTIVE (cpu_features, AMX_BF16);
 244           /* Determine if AMX_TILE is usable.  */
 245           CPU_FEATURE_SET_ACTIVE (cpu_features, AMX_TILE);
 246           /* Determine if AMX_INT8 is usable.  */
 247           CPU_FEATURE_SET_ACTIVE (cpu_features, AMX_INT8);
 248           /* Determine if AMX_FP16 is usable.  */
 249           CPU_FEATURE_SET_ACTIVE (cpu_features, AMX_FP16);
 250           /* Determine if AMX_COMPLEX is usable.  */
 251           CPU_FEATURE_SET_ACTIVE (cpu_features, AMX_COMPLEX);
 252         }
 253
 254       /* APX is usable only if the APX state is supported by kernel.  */
 255       if ((xcrlow & bit_APX_state) != 0)
 256         CPU_FEATURE_SET_ACTIVE (cpu_features, APX_F);
 257
 258       /* These features are usable only when OSXSAVE is enabled.  */
 259       CPU_FEATURE_SET (cpu_features, XSAVE);
 260       CPU_FEATURE_SET_ACTIVE (cpu_features, XSAVEOPT);
 261       CPU_FEATURE_SET_ACTIVE (cpu_features, XSAVEC);
 262       CPU_FEATURE_SET_ACTIVE (cpu_features, XGETBV_ECX_1);
 263       CPU_FEATURE_SET_ACTIVE (cpu_features, XFD);
 264
 265       /* For _dl_runtime_resolve, set xsave_state_size to xsave area
 266          size + integer register save size and align it to 64 bytes.  */
 267       if (cpu_features->basic.max_cpuid >= 0xd)
 268         {
 269           unsigned int eax, ebx, ecx, edx;
 270
 271           __cpuid_count (0xd, 0, eax, ebx, ecx, edx);
 272           if (ebx != 0)
 273             {
 274               unsigned int xsave_state_full_size
 275                 = ALIGN_UP (ebx + STATE_SAVE_OFFSET, 64);
 276
 277               cpu_features->xsave_state_size
 278                 = xsave_state_full_size;
 279               cpu_features->xsave_state_full_size
 280                 = xsave_state_full_size;
 281
 282               /* Check if XSAVEC is available.  */
 283               if (CPU_FEATURES_CPU_P (cpu_features, XSAVEC))
 284                 {
 285                   unsigned int xstate_comp_offsets[32];
 286                   unsigned int xstate_comp_sizes[32];
 287                   unsigned int i;
 288
 289                   xstate_comp_offsets[0] = 0;
 290                   xstate_comp_offsets[1] = 160;
 291                   xstate_comp_offsets[2] = 576;
 292                   xstate_comp_sizes[0] = 160;
 293                   xstate_comp_sizes[1] = 256;
 294
 295                   for (i = 2; i < 32; i++)
 296                     {
 297                       if ((STATE_SAVE_MASK & (1 << i)) != 0)
 298                         {
 299                           __cpuid_count (0xd, i, eax, ebx, ecx, edx);
 300                           xstate_comp_sizes[i] = eax;
 301                         }
 302                       else
 303                         {
 304                           ecx = 0;
 305                           xstate_comp_sizes[i] = 0;
 306                         }
 307
 308                       if (i > 2)
 309                         {
 310                           xstate_comp_offsets[i]
 311                             = (xstate_comp_offsets[i - 1]
 312                                + xstate_comp_sizes[i -1]);
 313                           if ((ecx & (1 << 1)) != 0)
 314                             xstate_comp_offsets[i]
 315                               = ALIGN_UP (xstate_comp_offsets[i], 64);
 316                         }
 317                     }
 318
 319                   /* Use XSAVEC.  */
 320                   unsigned int size
 321                     = xstate_comp_offsets[31] + xstate_comp_sizes[31];
 322                   if (size)
 323                     {
 324                       cpu_features->xsave_state_size
 325                         = ALIGN_UP (size + STATE_SAVE_OFFSET, 64);
 326                       CPU_FEATURE_SET (cpu_features, XSAVEC);
 327                     }
 328                 }
 329             }
 330         }
 331     }
 332
 333   /* Determine if PKU is usable.  */
 334   if (CPU_FEATURES_CPU_P (cpu_features, OSPKE))
 335     CPU_FEATURE_SET (cpu_features, PKU);
 336
 337   /* Determine if Key Locker instructions are usable.  */
 338   if (CPU_FEATURES_CPU_P (cpu_features, AESKLE))
 339     {
 340       CPU_FEATURE_SET (cpu_features, AESKLE);
 341       CPU_FEATURE_SET_ACTIVE (cpu_features, KL);
 342       CPU_FEATURE_SET_ACTIVE (cpu_features, WIDE_KL);
 343     }
 344
 345   dl_check_hwcap2 (cpu_features);
 346
 347   cpu_features->isa_1 = get_isa_level (cpu_features);
 348 }
 349
 350 static void
 351 get_extended_indices (struct cpu_features *cpu_features)
 352 {
 353   unsigned int eax, ebx, ecx, edx;
 354   __cpuid (0x80000000, eax, ebx, ecx, edx);
 355   if (eax >= 0x80000001)
 356     __cpuid (0x80000001,
 357              cpu_features->features[CPUID_INDEX_80000001].cpuid.eax,
 358              cpu_features->features[CPUID_INDEX_80000001].cpuid.ebx,
 359              cpu_features->features[CPUID_INDEX_80000001].cpuid.ecx,
 360              cpu_features->features[CPUID_INDEX_80000001].cpuid.edx);
 361   if (eax >= 0x80000007)
 362     __cpuid (0x80000007,
 363              cpu_features->features[CPUID_INDEX_80000007].cpuid.eax,
 364              cpu_features->features[CPUID_INDEX_80000007].cpuid.ebx,
 365              cpu_features->features[CPUID_INDEX_80000007].cpuid.ecx,
 366              cpu_features->features[CPUID_INDEX_80000007].cpuid.edx);
 367   if (eax >= 0x80000008)
 368     __cpuid (0x80000008,
 369              cpu_features->features[CPUID_INDEX_80000008].cpuid.eax,
 370              cpu_features->features[CPUID_INDEX_80000008].cpuid.ebx,
 371              cpu_features->features[CPUID_INDEX_80000008].cpuid.ecx,
 372              cpu_features->features[CPUID_INDEX_80000008].cpuid.edx);
 373 }
 374
 375 static void
 376 get_common_indices (struct cpu_features *cpu_features,
 377                     unsigned int *family, unsigned int *model,
 378                     unsigned int *extended_model, unsigned int *stepping)
 379 {
 380   if (family)
 381     {
 382       unsigned int eax;
 383       __cpuid (1, eax,
 384                cpu_features->features[CPUID_INDEX_1].cpuid.ebx,
 385                cpu_features->features[CPUID_INDEX_1].cpuid.ecx,
 386                cpu_features->features[CPUID_INDEX_1].cpuid.edx);
 387       cpu_features->features[CPUID_INDEX_1].cpuid.eax = eax;
 388       *family = (eax >> 8) & 0x0f;
 389       *model = (eax >> 4) & 0x0f;
 390       *extended_model = (eax >> 12) & 0xf0;
 391       *stepping = eax & 0x0f;
 392       if (*family == 0x0f)
 393         {
 394           *family += (eax >> 20) & 0xff;
 395           *model += *extended_model;
 396         }
 397     }
 398
 399   if (cpu_features->basic.max_cpuid >= 7)
 400     {
 401       __cpuid_count (7, 0,
 402                      cpu_features->features[CPUID_INDEX_7].cpuid.eax,
 403                      cpu_features->features[CPUID_INDEX_7].cpuid.ebx,
 404                      cpu_features->features[CPUID_INDEX_7].cpuid.ecx,
 405                      cpu_features->features[CPUID_INDEX_7].cpuid.edx);
 406       __cpuid_count (7, 1,
 407                      cpu_features->features[CPUID_INDEX_7_ECX_1].cpuid.eax,
 408                      cpu_features->features[CPUID_INDEX_7_ECX_1].cpuid.ebx,
 409                      cpu_features->features[CPUID_INDEX_7_ECX_1].cpuid.ecx,
 410                      cpu_features->features[CPUID_INDEX_7_ECX_1].cpuid.edx);
 411     }
 412
 413   if (cpu_features->basic.max_cpuid >= 0xd)
 414     __cpuid_count (0xd, 1,
 415                    cpu_features->features[CPUID_INDEX_D_ECX_1].cpuid.eax,
 416                    cpu_features->features[CPUID_INDEX_D_ECX_1].cpuid.ebx,
 417                    cpu_features->features[CPUID_INDEX_D_ECX_1].cpuid.ecx,
 418                    cpu_features->features[CPUID_INDEX_D_ECX_1].cpuid.edx);
 419
 420   if (cpu_features->basic.max_cpuid >= 0x14)
 421     __cpuid_count (0x14, 0,
 422                    cpu_features->features[CPUID_INDEX_14_ECX_0].cpuid.eax,
 423                    cpu_features->features[CPUID_INDEX_14_ECX_0].cpuid.ebx,
 424                    cpu_features->features[CPUID_INDEX_14_ECX_0].cpuid.ecx,
 425                    cpu_features->features[CPUID_INDEX_14_ECX_0].cpuid.edx);
 426
 427   if (cpu_features->basic.max_cpuid >= 0x19)
 428     __cpuid_count (0x19, 0,
 429                    cpu_features->features[CPUID_INDEX_19].cpuid.eax,
 430                    cpu_features->features[CPUID_INDEX_19].cpuid.ebx,
 431                    cpu_features->features[CPUID_INDEX_19].cpuid.ecx,
 432                    cpu_features->features[CPUID_INDEX_19].cpuid.edx);
 433
 434   dl_check_minsigstacksize (cpu_features);
 435 }
 436
 437 _Static_assert (((index_arch_Fast_Unaligned_Load
 438                   == index_arch_Fast_Unaligned_Copy)
 439                  && (index_arch_Fast_Unaligned_Load
 440                      == index_arch_Prefer_PMINUB_for_stringop)
 441                  && (index_arch_Fast_Unaligned_Load
 442                      == index_arch_Slow_SSE4_2)
 443                  && (index_arch_Fast_Unaligned_Load
 444                      == index_arch_Fast_Rep_String)
 445                  && (index_arch_Fast_Unaligned_Load
 446                      == index_arch_Fast_Copy_Backward)),
 447                 "Incorrect index_arch_Fast_Unaligned_Load");
 448
 449
 450 /* Intel Family-6 microarch list.  */
 451 enum
 452 {
 453   /* Atom processors.  */
 454   INTEL_ATOM_BONNELL,
 455   INTEL_ATOM_SILVERMONT,
 456   INTEL_ATOM_AIRMONT,
 457   INTEL_ATOM_GOLDMONT,
 458   INTEL_ATOM_GOLDMONT_PLUS,
 459   INTEL_ATOM_SIERRAFOREST,
 460   INTEL_ATOM_GRANDRIDGE,
 461   INTEL_ATOM_TREMONT,
 462
 463   /* Bigcore processors.  */
 464   INTEL_BIGCORE_MEROM,
 465   INTEL_BIGCORE_PENRYN,
 466   INTEL_BIGCORE_DUNNINGTON,
 467   INTEL_BIGCORE_NEHALEM,
 468   INTEL_BIGCORE_WESTMERE,
 469   INTEL_BIGCORE_SANDYBRIDGE,
 470   INTEL_BIGCORE_IVYBRIDGE,
 471   INTEL_BIGCORE_HASWELL,
 472   INTEL_BIGCORE_BROADWELL,
 473   INTEL_BIGCORE_SKYLAKE,
 474   INTEL_BIGCORE_KABYLAKE,
 475   INTEL_BIGCORE_COMETLAKE,
 476   INTEL_BIGCORE_SKYLAKE_AVX512,
 477   INTEL_BIGCORE_CANNONLAKE,
 478   INTEL_BIGCORE_ICELAKE,
 479   INTEL_BIGCORE_TIGERLAKE,
 480   INTEL_BIGCORE_ROCKETLAKE,
 481   INTEL_BIGCORE_SAPPHIRERAPIDS,
 482   INTEL_BIGCORE_RAPTORLAKE,
 483   INTEL_BIGCORE_EMERALDRAPIDS,
 484   INTEL_BIGCORE_METEORLAKE,
 485   INTEL_BIGCORE_LUNARLAKE,
 486   INTEL_BIGCORE_ARROWLAKE,
 487   INTEL_BIGCORE_GRANITERAPIDS,
 488
 489   /* Mixed (bigcore + atom SOC).  */
 490   INTEL_MIXED_LAKEFIELD,
 491   INTEL_MIXED_ALDERLAKE,
 492
 493   /* KNL.  */
 494   INTEL_KNIGHTS_MILL,
 495   INTEL_KNIGHTS_LANDING,
 496
 497   /* Unknown.  */
 498   INTEL_UNKNOWN,
 499 };
 500
 501 static unsigned int
 502 intel_get_fam6_microarch (unsigned int model,
 503                           __attribute__ ((unused)) unsigned int stepping)
 504 {
 505   switch (model)
 506     {
 507     case 0x1C:
 508     case 0x26:
 509       return INTEL_ATOM_BONNELL;
 510     case 0x27:
 511     case 0x35:
 512     case 0x36:
 513       /* Really Saltwell, but Saltwell is just a die shrink of Bonnell
 514          (microarchitecturally identical).  */
 515       return INTEL_ATOM_BONNELL;
 516     case 0x37:
 517     case 0x4A:
 518     case 0x4D:
 519     case 0x5D:
 520       return INTEL_ATOM_SILVERMONT;
 521     case 0x4C:
 522     case 0x5A:
 523     case 0x75:
 524       return INTEL_ATOM_AIRMONT;
 525     case 0x5C:
 526     case 0x5F:
 527       return INTEL_ATOM_GOLDMONT;
 528     case 0x7A:
 529       return INTEL_ATOM_GOLDMONT_PLUS;
 530     case 0xAF:
 531       return INTEL_ATOM_SIERRAFOREST;
 532     case 0xB6:
 533       return INTEL_ATOM_GRANDRIDGE;
 534     case 0x86:
 535     case 0x96:
 536     case 0x9C:
 537       return INTEL_ATOM_TREMONT;
 538     case 0x0F:
 539     case 0x16:
 540       return INTEL_BIGCORE_MEROM;
 541     case 0x17:
 542       return INTEL_BIGCORE_PENRYN;
 543     case 0x1D:
 544       return INTEL_BIGCORE_DUNNINGTON;
 545     case 0x1A:
 546     case 0x1E:
 547     case 0x1F:
 548     case 0x2E:
 549       return INTEL_BIGCORE_NEHALEM;
 550     case 0x25:
 551     case 0x2C:
 552     case 0x2F:
 553       return INTEL_BIGCORE_WESTMERE;
 554     case 0x2A:
 555     case 0x2D:
 556       return INTEL_BIGCORE_SANDYBRIDGE;
 557     case 0x3A:
 558     case 0x3E:
 559       return INTEL_BIGCORE_IVYBRIDGE;
 560     case 0x3C:
 561     case 0x3F:
 562     case 0x45:
 563     case 0x46:
 564       return INTEL_BIGCORE_HASWELL;
 565     case 0x3D:
 566     case 0x47:
 567     case 0x4F:
 568     case 0x56:
 569       return INTEL_BIGCORE_BROADWELL;
 570     case 0x4E:
 571     case 0x5E:
 572       return INTEL_BIGCORE_SKYLAKE;
 573     case 0x8E:
 574     /*
 575      Stepping = {9}
 576         -> Amberlake
 577      Stepping = {10}
 578         -> Coffeelake
 579      Stepping = {11, 12}
 580         -> Whiskeylake
 581      else
 582         -> Kabylake
 583
 584      All of these are derivatives of Kabylake (Skylake client).
 585      */
 586           return INTEL_BIGCORE_KABYLAKE;
 587     case 0x9E:
 588     /*
 589      Stepping = {10, 11, 12, 13}
 590         -> Coffeelake
 591      else
 592         -> Kabylake
 593
 594      Coffeelake is a derivatives of Kabylake (Skylake client).
 595      */
 596           return INTEL_BIGCORE_KABYLAKE;
 597     case 0xA5:
 598     case 0xA6:
 599       return INTEL_BIGCORE_COMETLAKE;
 600     case 0x66:
 601       return INTEL_BIGCORE_CANNONLAKE;
 602     case 0x55:
 603     /*
 604      Stepping = {6, 7}
 605         -> Cascadelake
 606      Stepping = {11}
 607         -> Cooperlake
 608      else
 609         -> Skylake-avx512
 610
 611      These are all microarchitecturally identical, so use
 612      Skylake-avx512 for all of them.
 613      */
 614       return INTEL_BIGCORE_SKYLAKE_AVX512;
 615     case 0x6A:
 616     case 0x6C:
 617     case 0x7D:
 618     case 0x7E:
 619     case 0x9D:
 620       return INTEL_BIGCORE_ICELAKE;
 621     case 0x8C:
 622     case 0x8D:
 623       return INTEL_BIGCORE_TIGERLAKE;
 624     case 0xA7:
 625       return INTEL_BIGCORE_ROCKETLAKE;
 626     case 0x8F:
 627       return INTEL_BIGCORE_SAPPHIRERAPIDS;
 628     case 0xB7:
 629     case 0xBA:
 630     case 0xBF:
 631       return INTEL_BIGCORE_RAPTORLAKE;
 632     case 0xCF:
 633       return INTEL_BIGCORE_EMERALDRAPIDS;
 634     case 0xAA:
 635     case 0xAC:
 636       return INTEL_BIGCORE_METEORLAKE;
 637     case 0xbd:
 638       return INTEL_BIGCORE_LUNARLAKE;
 639     case 0xc6:
 640       return INTEL_BIGCORE_ARROWLAKE;
 641     case 0xAD:
 642     case 0xAE:
 643       return INTEL_BIGCORE_GRANITERAPIDS;
 644     case 0x8A:
 645       return INTEL_MIXED_LAKEFIELD;
 646     case 0x97:
 647     case 0x9A:
 648     case 0xBE:
 649       return INTEL_MIXED_ALDERLAKE;
 650     case 0x85:
 651       return INTEL_KNIGHTS_MILL;
 652     case 0x57:
 653       return INTEL_KNIGHTS_LANDING;
 654     default:
 655       return INTEL_UNKNOWN;
 656     }
 657 }
 658
 659 static inline void
 660 init_cpu_features (struct cpu_features *cpu_features)
 661 {
 662   unsigned int ebx, ecx, edx;
 663   unsigned int family = 0;
 664   unsigned int model = 0;
 665   unsigned int stepping = 0;
 666   enum cpu_features_kind kind;
 667
 668   cpu_features->cachesize_non_temporal_divisor = 4;
 669 #if !HAS_CPUID
 670   if (__get_cpuid_max (0, 0) == 0)
 671     {
 672       kind = arch_kind_other;
 673       goto no_cpuid;
 674     }
 675 #endif
 676
 677   __cpuid (0, cpu_features->basic.max_cpuid, ebx, ecx, edx);
 678
 679   /* This spells out "GenuineIntel".  */
 680   if (ebx == 0x756e6547 && ecx == 0x6c65746e && edx == 0x49656e69)
 681     {
 682       unsigned int extended_model;
 683
 684       kind = arch_kind_intel;
 685
 686       get_common_indices (cpu_features, &family, &model, &extended_model,
 687                           &stepping);
 688
 689       get_extended_indices (cpu_features);
 690
 691       update_active (cpu_features);
 692
 693       if (family == 0x06)
 694         {
 695           model += extended_model;
 696           unsigned int microarch
 697               = intel_get_fam6_microarch (model, stepping);
 698
 699           switch (microarch)
 700             {
 701               /* Atom / KNL tuning.  */
 702             case INTEL_ATOM_BONNELL:
 703               /* BSF is slow on Bonnell.  */
 704               cpu_features->preferred[index_arch_Slow_BSF]
 705                   |= bit_arch_Slow_BSF;
 706               break;
 707
 708               /* Unaligned load versions are faster than SSSE3
 709                      on Airmont, Silvermont, Goldmont, and Goldmont Plus.  */
 710             case INTEL_ATOM_AIRMONT:
 711             case INTEL_ATOM_SILVERMONT:
 712             case INTEL_ATOM_GOLDMONT:
 713             case INTEL_ATOM_GOLDMONT_PLUS:
 714
 715           /* Knights Landing.  Enable Silvermont optimizations.  */
 716             case INTEL_KNIGHTS_LANDING:
 717
 718               cpu_features->preferred[index_arch_Fast_Unaligned_Load]
 719                   |= (bit_arch_Fast_Unaligned_Load
 720                       | bit_arch_Fast_Unaligned_Copy
 721                       | bit_arch_Prefer_PMINUB_for_stringop
 722                       | bit_arch_Slow_SSE4_2);
 723               break;
 724
 725             case INTEL_ATOM_TREMONT:
 726               /* Enable rep string instructions, unaligned load, unaligned
 727                  copy, pminub and avoid SSE 4.2 on Tremont.  */
 728               cpu_features->preferred[index_arch_Fast_Rep_String]
 729                   |= (bit_arch_Fast_Rep_String
 730                       | bit_arch_Fast_Unaligned_Load
 731                       | bit_arch_Fast_Unaligned_Copy
 732                       | bit_arch_Prefer_PMINUB_for_stringop
 733                       | bit_arch_Slow_SSE4_2);
 734               break;
 735
 736            /*
 737             Default tuned Knights microarch.
 738             case INTEL_KNIGHTS_MILL:
 739         */
 740
 741            /*
 742             Default tuned atom microarch.
 743             case INTEL_ATOM_SIERRAFOREST:
 744             case INTEL_ATOM_GRANDRIDGE:
 745            */
 746
 747               /* Bigcore/Default Tuning.  */
 748             default:
 749             default_tuning:
 750               /* Unknown family 0x06 processors.  Assuming this is one
 751                  of Core i3/i5/i7 processors if AVX is available.  */
 752               if (!CPU_FEATURES_CPU_P (cpu_features, AVX))
 753                 break;
 754
 755             enable_modern_features:
 756               /* Rep string instructions, unaligned load, unaligned copy,
 757                  and pminub are fast on Intel Core i3, i5 and i7.  */
 758               cpu_features->preferred[index_arch_Fast_Rep_String]
 759                   |= (bit_arch_Fast_Rep_String
 760                       | bit_arch_Fast_Unaligned_Load
 761                       | bit_arch_Fast_Unaligned_Copy
 762                       | bit_arch_Prefer_PMINUB_for_stringop);
 763               break;
 764
 765             case INTEL_BIGCORE_NEHALEM:
 766             case INTEL_BIGCORE_WESTMERE:
 767               /* Older CPUs prefer non-temporal stores at lower threshold.  */
 768               cpu_features->cachesize_non_temporal_divisor = 8;
 769               goto enable_modern_features;
 770
 771               /* Older Bigcore microarch (smaller non-temporal store
 772                  threshold).  */
 773             case INTEL_BIGCORE_SANDYBRIDGE:
 774             case INTEL_BIGCORE_IVYBRIDGE:
 775             case INTEL_BIGCORE_HASWELL:
 776             case INTEL_BIGCORE_BROADWELL:
 777               cpu_features->cachesize_non_temporal_divisor = 8;
 778               goto default_tuning;
 779
 780               /* Newer Bigcore microarch (larger non-temporal store
 781                  threshold).  */
 782             case INTEL_BIGCORE_SKYLAKE:
 783             case INTEL_BIGCORE_KABYLAKE:
 784             case INTEL_BIGCORE_COMETLAKE:
 785             case INTEL_BIGCORE_SKYLAKE_AVX512:
 786             case INTEL_BIGCORE_CANNONLAKE:
 787             case INTEL_BIGCORE_ICELAKE:
 788             case INTEL_BIGCORE_TIGERLAKE:
 789             case INTEL_BIGCORE_ROCKETLAKE:
 790             case INTEL_BIGCORE_RAPTORLAKE:
 791             case INTEL_BIGCORE_METEORLAKE:
 792             case INTEL_BIGCORE_LUNARLAKE:
 793             case INTEL_BIGCORE_ARROWLAKE:
 794             case INTEL_BIGCORE_SAPPHIRERAPIDS:
 795             case INTEL_BIGCORE_EMERALDRAPIDS:
 796             case INTEL_BIGCORE_GRANITERAPIDS:
 797               cpu_features->cachesize_non_temporal_divisor = 2;
 798               goto default_tuning;
 799
 800               /* Default tuned Mixed (bigcore + atom SOC). */
 801             case INTEL_MIXED_LAKEFIELD:
 802             case INTEL_MIXED_ALDERLAKE:
 803               cpu_features->cachesize_non_temporal_divisor = 2;
 804               goto default_tuning;
 805             }
 806
 807               /* Disable TSX on some processors to avoid TSX on kernels that
 808                  weren't updated with the latest microcode package (which
 809                  disables broken feature by default).  */
 810           switch (microarch)
 811             {
 812             case INTEL_BIGCORE_SKYLAKE_AVX512:
 813               /* 0x55 (Skylake-avx512) && stepping <= 5 disable TSX. */
 814               if (stepping <= 5)
 815                 goto disable_tsx;
 816               break;
 817
 818             case INTEL_BIGCORE_KABYLAKE:
 819               /* NB: Although the errata documents that for model == 0x8e
 820                      (kabylake skylake client), only 0xb stepping or lower are
 821                      impacted, the intention of the errata was to disable TSX on
 822                      all client processors on all steppings.  Include 0xc
 823                      stepping which is an Intel Core i7-8665U, a client mobile
 824                      processor.  */
 825               if (stepping > 0xc)
 826                 break;
 827               /* Fall through.  */
 828             case INTEL_BIGCORE_SKYLAKE:
 829                 /* Disable Intel TSX and enable RTM_ALWAYS_ABORT for
 830                    processors listed in:
 831
 832 https://www.intel.com/content/www/us/en/support/articles/000059422/processors.html
 833                  */
 834             disable_tsx:
 835                 CPU_FEATURE_UNSET (cpu_features, HLE);
 836                 CPU_FEATURE_UNSET (cpu_features, RTM);
 837                 CPU_FEATURE_SET (cpu_features, RTM_ALWAYS_ABORT);
 838                 break;
 839
 840             case INTEL_BIGCORE_HASWELL:
 841                 /* Xeon E7 v3 (model == 0x3f) with stepping >= 4 has working
 842                    TSX.  Haswell also include other model numbers that have
 843                    working TSX.  */
 844                 if (model == 0x3f && stepping >= 4)
 845                 break;
 846
 847                 CPU_FEATURE_UNSET (cpu_features, RTM);
 848                 break;
 849             }
 850         }
 851
 852
 853       /* Since AVX512ER is unique to Xeon Phi, set Prefer_No_VZEROUPPER
 854          if AVX512ER is available.  Don't use AVX512 to avoid lower CPU
 855          frequency if AVX512ER isn't available.  */
 856       if (CPU_FEATURES_CPU_P (cpu_features, AVX512ER))
 857         cpu_features->preferred[index_arch_Prefer_No_VZEROUPPER]
 858           |= bit_arch_Prefer_No_VZEROUPPER;
 859       else
 860         {
 861           /* Processors with AVX512 and AVX-VNNI won't lower CPU frequency
 862              when ZMM load and store instructions are used.  */
 863           if (!CPU_FEATURES_CPU_P (cpu_features, AVX_VNNI))
 864             cpu_features->preferred[index_arch_Prefer_No_AVX512]
 865               |= bit_arch_Prefer_No_AVX512;
 866
 867           /* Avoid RTM abort triggered by VZEROUPPER inside a
 868              transactionally executing RTM region.  */
 869           if (CPU_FEATURE_USABLE_P (cpu_features, RTM))
 870             cpu_features->preferred[index_arch_Prefer_No_VZEROUPPER]
 871               |= bit_arch_Prefer_No_VZEROUPPER;
 872         }
 873
 874       /* Avoid avoid short distance REP MOVSB on processor with FSRM.  */
 875       if (CPU_FEATURES_CPU_P (cpu_features, FSRM))
 876         cpu_features->preferred[index_arch_Avoid_Short_Distance_REP_MOVSB]
 877           |= bit_arch_Avoid_Short_Distance_REP_MOVSB;
 878     }
 879   /* This spells out "AuthenticAMD" or "HygonGenuine".  */
 880   else if ((ebx == 0x68747541 && ecx == 0x444d4163 && edx == 0x69746e65)
 881            || (ebx == 0x6f677948 && ecx == 0x656e6975 && edx == 0x6e65476e))
 882     {
 883       unsigned int extended_model;
 884
 885       kind = arch_kind_amd;
 886
 887       get_common_indices (cpu_features, &family, &model, &extended_model,
 888                           &stepping);
 889
 890       get_extended_indices (cpu_features);
 891
 892       update_active (cpu_features);
 893
 894       ecx = cpu_features->features[CPUID_INDEX_1].cpuid.ecx;
 895
 896       if (CPU_FEATURE_USABLE_P (cpu_features, AVX))
 897         {
 898           /* Since the FMA4 bit is in CPUID_INDEX_80000001 and
 899              FMA4 requires AVX, determine if FMA4 is usable here.  */
 900           CPU_FEATURE_SET_ACTIVE (cpu_features, FMA4);
 901         }
 902
 903       if (family == 0x15)
 904         {
 905           /* "Excavator"   */
 906           if (model >= 0x60 && model <= 0x7f)
 907           {
 908             cpu_features->preferred[index_arch_Fast_Unaligned_Load]
 909               |= (bit_arch_Fast_Unaligned_Load
 910                   | bit_arch_Fast_Copy_Backward);
 911
 912             /* Unaligned AVX loads are slower.*/
 913             cpu_features->preferred[index_arch_AVX_Fast_Unaligned_Load]
 914               &= ~bit_arch_AVX_Fast_Unaligned_Load;
 915           }
 916         }
 917     }
 918   /* This spells out "CentaurHauls" or " Shanghai ".  */
 919   else if ((ebx == 0x746e6543 && ecx == 0x736c7561 && edx == 0x48727561)
 920            || (ebx == 0x68532020 && ecx == 0x20206961 && edx == 0x68676e61))
 921     {
 922       unsigned int extended_model, stepping;
 923
 924       kind = arch_kind_zhaoxin;
 925
 926       get_common_indices (cpu_features, &family, &model, &extended_model,
 927                           &stepping);
 928
 929       get_extended_indices (cpu_features);
 930
 931       update_active (cpu_features);
 932
 933       model += extended_model;
 934       if (family == 0x6)
 935         {
 936           if (model == 0xf || model == 0x19)
 937             {
 938               CPU_FEATURE_UNSET (cpu_features, AVX);
 939               CPU_FEATURE_UNSET (cpu_features, AVX2);
 940
 941               cpu_features->preferred[index_arch_Slow_SSE4_2]
 942                 |= bit_arch_Slow_SSE4_2;
 943
 944               cpu_features->preferred[index_arch_AVX_Fast_Unaligned_Load]
 945                 &= ~bit_arch_AVX_Fast_Unaligned_Load;
 946             }
 947         }
 948       else if (family == 0x7)
 949         {
 950           if (model == 0x1b)
 951             {
 952               CPU_FEATURE_UNSET (cpu_features, AVX);
 953               CPU_FEATURE_UNSET (cpu_features, AVX2);
 954
 955               cpu_features->preferred[index_arch_Slow_SSE4_2]
 956                 |= bit_arch_Slow_SSE4_2;
 957
 958               cpu_features->preferred[index_arch_AVX_Fast_Unaligned_Load]
 959                 &= ~bit_arch_AVX_Fast_Unaligned_Load;
 960             }
 961           else if (model == 0x3b)
 962             {
 963               CPU_FEATURE_UNSET (cpu_features, AVX);
 964               CPU_FEATURE_UNSET (cpu_features, AVX2);
 965
 966               cpu_features->preferred[index_arch_AVX_Fast_Unaligned_Load]
 967                 &= ~bit_arch_AVX_Fast_Unaligned_Load;
 968             }
 969         }
 970     }
 971   else
 972     {
 973       kind = arch_kind_other;
 974       get_common_indices (cpu_features, NULL, NULL, NULL, NULL);
 975       update_active (cpu_features);
 976     }
 977
 978   /* Support i586 if CX8 is available.  */
 979   if (CPU_FEATURES_CPU_P (cpu_features, CX8))
 980     cpu_features->preferred[index_arch_I586] |= bit_arch_I586;
 981
 982   /* Support i686 if CMOV is available.  */
 983   if (CPU_FEATURES_CPU_P (cpu_features, CMOV))
 984     cpu_features->preferred[index_arch_I686] |= bit_arch_I686;
 985
 986 #if !HAS_CPUID
 987 no_cpuid:
 988 #endif
 989
 990   cpu_features->basic.kind = kind;
 991   cpu_features->basic.family = family;
 992   cpu_features->basic.model = model;
 993   cpu_features->basic.stepping = stepping;
 994
 995   dl_init_cacheinfo (cpu_features);
 996
 997   TUNABLE_GET (hwcaps, tunable_val_t *, TUNABLE_CALLBACK (set_hwcaps));
 998
 999 #ifdef __LP64__
1000   TUNABLE_GET (prefer_map_32bit_exec, tunable_val_t *,
1001                TUNABLE_CALLBACK (set_prefer_map_32bit_exec));
1002 #endif
1003
1004   bool disable_xsave_features = false;
1005
1006   if (!CPU_FEATURE_USABLE_P (cpu_features, OSXSAVE))
1007     {
1008       /* These features are usable only if OSXSAVE is usable.  */
1009       CPU_FEATURE_UNSET (cpu_features, XSAVE);
1010       CPU_FEATURE_UNSET (cpu_features, XSAVEOPT);
1011       CPU_FEATURE_UNSET (cpu_features, XSAVEC);
1012       CPU_FEATURE_UNSET (cpu_features, XGETBV_ECX_1);
1013       CPU_FEATURE_UNSET (cpu_features, XFD);
1014
1015       disable_xsave_features = true;
1016     }
1017
1018   if (disable_xsave_features
1019       || (!CPU_FEATURE_USABLE_P (cpu_features, XSAVE)
1020           && !CPU_FEATURE_USABLE_P (cpu_features, XSAVEC)))
1021     {
1022       /* Clear xsave_state_size if both XSAVE and XSAVEC aren't usable.  */
1023       cpu_features->xsave_state_size = 0;
1024
1025       CPU_FEATURE_UNSET (cpu_features, AVX);
1026       CPU_FEATURE_UNSET (cpu_features, AVX2);
1027       CPU_FEATURE_UNSET (cpu_features, AVX_VNNI);
1028       CPU_FEATURE_UNSET (cpu_features, FMA);
1029       CPU_FEATURE_UNSET (cpu_features, VAES);
1030       CPU_FEATURE_UNSET (cpu_features, VPCLMULQDQ);
1031       CPU_FEATURE_UNSET (cpu_features, XOP);
1032       CPU_FEATURE_UNSET (cpu_features, F16C);
1033       CPU_FEATURE_UNSET (cpu_features, AVX512F);
1034       CPU_FEATURE_UNSET (cpu_features, AVX512CD);
1035       CPU_FEATURE_UNSET (cpu_features, AVX512ER);
1036       CPU_FEATURE_UNSET (cpu_features, AVX512PF);
1037       CPU_FEATURE_UNSET (cpu_features, AVX512VL);
1038       CPU_FEATURE_UNSET (cpu_features, AVX512DQ);
1039       CPU_FEATURE_UNSET (cpu_features, AVX512BW);
1040       CPU_FEATURE_UNSET (cpu_features, AVX512_4FMAPS);
1041       CPU_FEATURE_UNSET (cpu_features, AVX512_4VNNIW);
1042       CPU_FEATURE_UNSET (cpu_features, AVX512_BITALG);
1043       CPU_FEATURE_UNSET (cpu_features, AVX512_IFMA);
1044       CPU_FEATURE_UNSET (cpu_features, AVX512_VBMI);
1045       CPU_FEATURE_UNSET (cpu_features, AVX512_VBMI2);
1046       CPU_FEATURE_UNSET (cpu_features, AVX512_VNNI);
1047       CPU_FEATURE_UNSET (cpu_features, AVX512_VPOPCNTDQ);
1048       CPU_FEATURE_UNSET (cpu_features, AVX512_VP2INTERSECT);
1049       CPU_FEATURE_UNSET (cpu_features, AVX512_BF16);
1050       CPU_FEATURE_UNSET (cpu_features, AVX512_FP16);
1051       CPU_FEATURE_UNSET (cpu_features, AMX_BF16);
1052       CPU_FEATURE_UNSET (cpu_features, AMX_TILE);
1053       CPU_FEATURE_UNSET (cpu_features, AMX_INT8);
1054
1055       CPU_FEATURE_UNSET (cpu_features, FMA4);
1056     }
1057
1058 #ifdef __x86_64__
1059   GLRO(dl_hwcap) = HWCAP_X86_64;
1060   if (cpu_features->basic.kind == arch_kind_intel)
1061     {
1062       const char *platform = NULL;
1063
1064       if (CPU_FEATURE_USABLE_P (cpu_features, AVX512CD))
1065         {
1066           if (CPU_FEATURE_USABLE_P (cpu_features, AVX512ER))
1067             {
1068               if (CPU_FEATURE_USABLE_P (cpu_features, AVX512PF))
1069                 platform = "xeon_phi";
1070             }
1071           else
1072             {
1073               if (CPU_FEATURE_USABLE_P (cpu_features, AVX512BW)
1074                   && CPU_FEATURE_USABLE_P (cpu_features, AVX512DQ)
1075                   && CPU_FEATURE_USABLE_P (cpu_features, AVX512VL))
1076                 GLRO(dl_hwcap) |= HWCAP_X86_AVX512_1;
1077             }
1078         }
1079
1080       if (platform == NULL
1081           && CPU_FEATURE_USABLE_P (cpu_features, AVX2)
1082           && CPU_FEATURE_USABLE_P (cpu_features, FMA)
1083           && CPU_FEATURE_USABLE_P (cpu_features, BMI1)
1084           && CPU_FEATURE_USABLE_P (cpu_features, BMI2)
1085           && CPU_FEATURE_USABLE_P (cpu_features, LZCNT)
1086           && CPU_FEATURE_USABLE_P (cpu_features, MOVBE)
1087           && CPU_FEATURE_USABLE_P (cpu_features, POPCNT))
1088         platform = "haswell";
1089
1090       if (platform != NULL)
1091         GLRO(dl_platform) = platform;
1092     }
1093 #else
1094   GLRO(dl_hwcap) = 0;
1095   if (CPU_FEATURE_USABLE_P (cpu_features, SSE2))
1096     GLRO(dl_hwcap) |= HWCAP_X86_SSE2;
1097
1098   if (CPU_FEATURES_ARCH_P (cpu_features, I686))
1099     GLRO(dl_platform) = "i686";
1100   else if (CPU_FEATURES_ARCH_P (cpu_features, I586))
1101     GLRO(dl_platform) = "i586";
1102 #endif
1103
1104 #if CET_ENABLED
1105   TUNABLE_GET (x86_ibt, tunable_val_t *,
1106                TUNABLE_CALLBACK (set_x86_ibt));
1107   TUNABLE_GET (x86_shstk, tunable_val_t *,
1108                TUNABLE_CALLBACK (set_x86_shstk));
1109
1110   /* Check CET status.  */
1111   unsigned int cet_status = get_cet_status ();
1112
1113   if ((cet_status & GNU_PROPERTY_X86_FEATURE_1_IBT) == 0)
1114     CPU_FEATURE_UNSET (cpu_features, IBT)
1115   if ((cet_status & GNU_PROPERTY_X86_FEATURE_1_SHSTK) == 0)
1116     CPU_FEATURE_UNSET (cpu_features, SHSTK)
1117
1118   if (cet_status)
1119     {
1120       GL(dl_x86_feature_1) = cet_status;
1121
1122 # ifndef SHARED
1123       /* Check if IBT and SHSTK are enabled by kernel.  */
1124       if ((cet_status & GNU_PROPERTY_X86_FEATURE_1_IBT)
1125           || (cet_status & GNU_PROPERTY_X86_FEATURE_1_SHSTK))
1126         {
1127           /* Disable IBT and/or SHSTK if they are enabled by kernel, but
1128              disabled by environment variable:
1129
1130              GLIBC_TUNABLES=glibc.cpu.hwcaps=-IBT,-SHSTK
1131            */
1132           unsigned int cet_feature = 0;
1133           if (!CPU_FEATURE_USABLE (IBT))
1134             cet_feature |= GNU_PROPERTY_X86_FEATURE_1_IBT;
1135           if (!CPU_FEATURE_USABLE (SHSTK))
1136             cet_feature |= GNU_PROPERTY_X86_FEATURE_1_SHSTK;
1137
1138           if (cet_feature)
1139             {
1140               int res = dl_cet_disable_cet (cet_feature);
1141
1142               /* Clear the disabled bits in dl_x86_feature_1.  */
1143               if (res == 0)
1144                 GL(dl_x86_feature_1) &= ~cet_feature;
1145             }
1146
1147           /* Lock CET if IBT or SHSTK is enabled in executable.  Don't
1148              lock CET if IBT or SHSTK is enabled permissively.  */
1149           if (GL(dl_x86_feature_control).ibt != cet_permissive
1150               && GL(dl_x86_feature_control).shstk != cet_permissive)
1151             dl_cet_lock_cet ();
1152         }
1153 # endif
1154     }
1155 #endif
1156
1157 #ifndef SHARED
1158   /* NB: In libc.a, call init_cacheinfo.  */
1159   init_cacheinfo ();
1160 #endif
1161 }