sysdeps/unix/sysv/linux/aarch64/cpu-features.c

   1 /* Initialize CPU feature data.  AArch64 version.
   2    This file is part of the GNU C Library.
   3    Copyright (C) 2017-2024 Free Software Foundation, Inc.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <https://www.gnu.org/licenses/>.  */
  18
  19 #include <array_length.h>
  20 #include <cpu-features.h>
  21 #include <sys/auxv.h>
  22 #include <elf/dl-hwcaps.h>
  23 #include <sys/prctl.h>
  24 #include <sys/utsname.h>
  25 #include <dl-tunables-parse.h>
  26
  27 #define DCZID_DZP_MASK (1 << 4)
  28 #define DCZID_BS_MASK (0xf)
  29
  30 /* The maximal set of permitted tags that the MTE random tag generation
  31    instruction may use.  We exclude tag 0 because a) we want to reserve
  32    that for the libc heap structures and b) because it makes it easier
  33    to see when pointer have been correctly tagged.  */
  34 #define MTE_ALLOWED_TAGS (0xfffe << PR_MTE_TAG_SHIFT)
  35
  36 struct cpu_list
  37 {
  38   const char *name;
  39   size_t len;
  40   uint64_t midr;
  41 };
  42
  43 static const struct cpu_list cpu_list[] =
  44 {
  45 #define CPU_LIST_ENTRY(__str, __num) { __str, sizeof (__str) - 1, __num }
  46   CPU_LIST_ENTRY ("thunderxt88",    0x430F0A10),
  47   CPU_LIST_ENTRY ("thunderx2t99",   0x431F0AF0),
  48   CPU_LIST_ENTRY ("thunderx2t99p1", 0x420F5160),
  49   CPU_LIST_ENTRY ("ares",           0x411FD0C0),
  50   CPU_LIST_ENTRY ("emag",           0x503F0001),
  51   CPU_LIST_ENTRY ("kunpeng920",     0x481FD010),
  52   CPU_LIST_ENTRY ("a64fx",          0x460F0010),
  53   CPU_LIST_ENTRY ("generic",        0x0),
  54 };
  55
  56 static uint64_t
  57 get_midr_from_mcpu (const struct tunable_str_t *mcpu)
  58 {
  59   for (int i = 0; i < array_length (cpu_list); i++)
  60     if (tunable_strcmp (mcpu, cpu_list[i].name, cpu_list[i].len))
  61       return cpu_list[i].midr;
  62
  63   return UINT64_MAX;
  64 }
  65
  66 #if __LINUX_KERNEL_VERSION < 0x060200
  67
  68 /* Return true if we prefer using SVE in string ifuncs.  Old kernels disable
  69    SVE after every system call which results in unnecessary traps if memcpy
  70    uses SVE.  This is true for kernels between 4.15.0 and before 6.2.0, except
  71    for 5.14.0 which was patched.  For these versions return false to avoid using
  72    SVE ifuncs.
  73    Parse the kernel version into a 24-bit kernel.major.minor value without
  74    calling any library functions.  If uname() is not supported or if the version
  75    format is not recognized, assume the kernel is modern and return true.  */
  76
  77 static inline bool
  78 prefer_sve_ifuncs (void)
  79 {
  80   struct utsname buf;
  81   const char *p = &buf.release[0];
  82   int kernel = 0;
  83   int val;
  84
  85   if (__uname (&buf) < 0)
  86     return true;
  87
  88   for (int shift = 16; shift >= 0; shift -= 8)
  89     {
  90       for (val = 0; *p >= '0' && *p <= '9'; p++)
  91         val = val * 10 + *p - '0';
  92       kernel |= (val & 255) << shift;
  93       if (*p++ != '.')
  94         break;
  95     }
  96
  97   if (kernel >= 0x060200 || kernel == 0x050e00)
  98     return true;
  99   if (kernel >= 0x040f00)
 100     return false;
 101   return true;
 102 }
 103
 104 #endif
 105
 106 static inline void
 107 init_cpu_features (struct cpu_features *cpu_features)
 108 {
 109   register uint64_t midr = UINT64_MAX;
 110
 111   /* Get the tunable override.  */
 112   const struct tunable_str_t *mcpu = TUNABLE_GET (glibc, cpu, name,
 113                                                   struct tunable_str_t *,
 114                                                   NULL);
 115   if (mcpu != NULL)
 116     midr = get_midr_from_mcpu (mcpu);
 117
 118   /* If there was no useful tunable override, query the MIDR if the kernel
 119      allows it.  */
 120   if (midr == UINT64_MAX)
 121     {
 122       if (GLRO (dl_hwcap) & HWCAP_CPUID)
 123         asm volatile ("mrs %0, midr_el1" : "=r"(midr));
 124       else
 125         midr = 0;
 126     }
 127
 128   cpu_features->midr_el1 = midr;
 129
 130   /* Check if ZVA is enabled.  */
 131   unsigned dczid;
 132   asm volatile ("mrs %0, dczid_el0" : "=r"(dczid));
 133
 134   if ((dczid & DCZID_DZP_MASK) == 0)
 135     cpu_features->zva_size = 4 << (dczid & DCZID_BS_MASK);
 136
 137   /* Check if BTI is supported.  */
 138   cpu_features->bti = GLRO (dl_hwcap2) & HWCAP2_BTI;
 139
 140   /* Setup memory tagging support if the HW and kernel support it, and if
 141      the user has requested it.  */
 142   cpu_features->mte_state = 0;
 143
 144 #ifdef USE_MTAG
 145   int mte_state = TUNABLE_GET (glibc, mem, tagging, unsigned, 0);
 146   cpu_features->mte_state = (GLRO (dl_hwcap2) & HWCAP2_MTE) ? mte_state : 0;
 147   /* If we lack the MTE feature, disable the tunable, since it will
 148      otherwise cause instructions that won't run on this CPU to be used.  */
 149   TUNABLE_SET (glibc, mem, tagging, cpu_features->mte_state);
 150
 151   if (cpu_features->mte_state & 4)
 152     /* Enable choosing system-preferred faulting mode.  */
 153     __prctl (PR_SET_TAGGED_ADDR_CTRL,
 154              (PR_TAGGED_ADDR_ENABLE | PR_MTE_TCF_SYNC | PR_MTE_TCF_ASYNC
 155               | MTE_ALLOWED_TAGS),
 156              0, 0, 0);
 157   else if (cpu_features->mte_state & 2)
 158     __prctl (PR_SET_TAGGED_ADDR_CTRL,
 159              (PR_TAGGED_ADDR_ENABLE | PR_MTE_TCF_SYNC | MTE_ALLOWED_TAGS),
 160              0, 0, 0);
 161   else if (cpu_features->mte_state)
 162     __prctl (PR_SET_TAGGED_ADDR_CTRL,
 163              (PR_TAGGED_ADDR_ENABLE | PR_MTE_TCF_ASYNC | MTE_ALLOWED_TAGS),
 164              0, 0, 0);
 165 #endif
 166
 167   /* Check if SVE is supported.  */
 168   cpu_features->sve = GLRO (dl_hwcap) & HWCAP_SVE;
 169
 170   cpu_features->prefer_sve_ifuncs = cpu_features->sve;
 171
 172 #if __LINUX_KERNEL_VERSION < 0x060200
 173   if (cpu_features->sve)
 174     cpu_features->prefer_sve_ifuncs = prefer_sve_ifuncs ();
 175 #endif
 176
 177   /* Check if MOPS is supported.  */
 178   cpu_features->mops = GLRO (dl_hwcap2) & HWCAP2_MOPS;
 179 }