sysdeps/unix/sysv/linux/malloc-hugepages.c

   1 /* Huge Page support.  Linux implementation.
   2    Copyright (C) 2021-2024 Free Software Foundation, Inc.
   3    This file is part of the GNU C Library.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public License as
   7    published by the Free Software Foundation; either version 2.1 of the
   8    License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; see the file COPYING.LIB.  If
  17    not, see <https://www.gnu.org/licenses/>.  */
  18
  19 #include <intprops.h>
  20 #include <dirent.h>
  21 #include <malloc-hugepages.h>
  22 #include <not-cancel.h>
  23 #include <sys/mman.h>
  24
  25 unsigned long int
  26 __malloc_default_thp_pagesize (void)
  27 {
  28   int fd = __open64_nocancel (
  29     "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size", O_RDONLY);
  30   if (fd == -1)
  31     return 0;
  32
  33   char str[INT_BUFSIZE_BOUND (unsigned long int)];
  34   ssize_t s = __read_nocancel (fd, str, sizeof (str));
  35   __close_nocancel (fd);
  36   if (s < 0)
  37     return 0;
  38
  39   unsigned long int r = 0;
  40   for (ssize_t i = 0; i < s; i++)
  41     {
  42       if (str[i] == '\n')
  43         break;
  44       r *= 10;
  45       r += str[i] - '0';
  46     }
  47   return r;
  48 }
  49
  50 enum malloc_thp_mode_t
  51 __malloc_thp_mode (void)
  52 {
  53   int fd = __open64_nocancel ("/sys/kernel/mm/transparent_hugepage/enabled",
  54                               O_RDONLY);
  55   if (fd == -1)
  56     return malloc_thp_mode_not_supported;
  57
  58   static const char mode_always[]  = "[always] madvise never\n";
  59   static const char mode_madvise[] = "always [madvise] never\n";
  60   static const char mode_never[]   = "always madvise [never]\n";
  61
  62   char str[sizeof(mode_always)];
  63   ssize_t s = __read_nocancel (fd, str, sizeof (str));
  64   if (s >= sizeof str || s < 0)
  65     return malloc_thp_mode_not_supported;
  66   str[s] = '\0';
  67   __close_nocancel (fd);
  68
  69   if (s == sizeof (mode_always) - 1)
  70     {
  71       if (strcmp (str, mode_always) == 0)
  72         return malloc_thp_mode_always;
  73       else if (strcmp (str, mode_madvise) == 0)
  74         return malloc_thp_mode_madvise;
  75       else if (strcmp (str, mode_never) == 0)
  76         return malloc_thp_mode_never;
  77     }
  78   return malloc_thp_mode_not_supported;
  79 }
  80
  81 static size_t
  82 malloc_default_hugepage_size (void)
  83 {
  84   int fd = __open64_nocancel ("/proc/meminfo", O_RDONLY);
  85   if (fd == -1)
  86     return 0;
  87
  88   size_t hpsize = 0;
  89
  90   char buf[512];
  91   off64_t off = 0;
  92   while (1)
  93     {
  94       ssize_t r = __pread64_nocancel (fd, buf, sizeof (buf) - 1, off);
  95       if (r < 0)
  96         break;
  97       buf[r] = '\0';
  98
  99       /* If the tag is not found, read the last line again.  */
 100       const char *s = strstr (buf, "Hugepagesize:");
 101       if (s == NULL)
 102         {
 103           char *nl = strrchr (buf, '\n');
 104           if (nl == NULL)
 105             break;
 106           off += (nl + 1) - buf;
 107           continue;
 108         }
 109
 110       /* The default huge page size is in the form:
 111          Hugepagesize:       NUMBER kB  */
 112       s += sizeof ("Hugepagesize: ") - 1;
 113       for (int i = 0; (s[i] >= '0' && s[i] <= '9') || s[i] == ' '; i++)
 114         {
 115           if (s[i] == ' ')
 116             continue;
 117           hpsize *= 10;
 118           hpsize += s[i] - '0';
 119         }
 120       hpsize *= 1024;
 121       break;
 122     }
 123
 124   __close_nocancel (fd);
 125
 126   return hpsize;
 127 }
 128
 129 static inline int
 130 hugepage_flags (size_t pagesize)
 131 {
 132   return MAP_HUGETLB | (__builtin_ctzll (pagesize) << MAP_HUGE_SHIFT);
 133 }
 134
 135 void
 136 __malloc_hugepage_config (size_t requested, size_t *pagesize, int *flags)
 137 {
 138   *pagesize = 0;
 139   *flags = 0;
 140
 141   if (requested == 0)
 142     {
 143       *pagesize = malloc_default_hugepage_size ();
 144       if (*pagesize != 0)
 145         *flags = hugepage_flags (*pagesize);
 146       return;
 147     }
 148
 149   /* Each entry represents a supported huge page in the form of:
 150      hugepages-<size>kB.  */
 151   int dirfd = __open64_nocancel ("/sys/kernel/mm/hugepages",
 152                                  O_RDONLY | O_DIRECTORY, 0);
 153   if (dirfd == -1)
 154     return;
 155
 156   char buffer[1024];
 157   while (true)
 158     {
 159 #if !IS_IN(libc)
 160 # define __getdents64 getdents64
 161 #endif
 162       ssize_t ret = __getdents64 (dirfd, buffer, sizeof (buffer));
 163       if (ret == -1)
 164         break;
 165       else if (ret == 0)
 166         break;
 167
 168       bool found = false;
 169       char *begin = buffer, *end = buffer + ret;
 170       while (begin != end)
 171         {
 172           unsigned short int d_reclen;
 173           memcpy (&d_reclen, begin + offsetof (struct dirent64, d_reclen),
 174                   sizeof (d_reclen));
 175           const char *dname = begin + offsetof (struct dirent64, d_name);
 176           begin += d_reclen;
 177
 178           if (dname[0] == '.'
 179               || strncmp (dname, "hugepages-", sizeof ("hugepages-") - 1) != 0)
 180             continue;
 181
 182           size_t hpsize = 0;
 183           const char *sizestr = dname + sizeof ("hugepages-") - 1;
 184           for (int i = 0; sizestr[i] >= '0' && sizestr[i] <= '9'; i++)
 185             {
 186               hpsize *= 10;
 187               hpsize += sizestr[i] - '0';
 188             }
 189           hpsize *= 1024;
 190
 191           if (hpsize == requested)
 192             {
 193               *pagesize = hpsize;
 194               *flags = hugepage_flags (*pagesize);
 195               found = true;
 196               break;
 197             }
 198         }
 199       if (found)
 200         break;
 201     }
 202
 203   __close_nocancel (dirfd);
 204 }