sysdeps/unix/sysv/linux/malloc-hugepages.c

   1 /* Huge Page support.  Linux implementation.
   2    Copyright (C) 2021-2023 Free Software Foundation, Inc.
   3    This file is part of the GNU C Library.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public License as
   7    published by the Free Software Foundation; either version 2.1 of the
   8    License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; see the file COPYING.LIB.  If
  17    not, see <https://www.gnu.org/licenses/>.  */
  18
  19 #include <intprops.h>
  20 #include <dirent.h>
  21 #include <malloc-hugepages.h>
  22 #include <not-cancel.h>
  23 #include <sys/mman.h>
  24
  25 unsigned long int
  26 __malloc_default_thp_pagesize (void)
  27 {
  28   int fd = __open64_nocancel (
  29     "/sys/kernel/mm/transparent_hugepage/hpage_pmd_size", O_RDONLY);
  30   if (fd == -1)
  31     return 0;
  32
  33   char str[INT_BUFSIZE_BOUND (unsigned long int)];
  34   ssize_t s = __read_nocancel (fd, str, sizeof (str));
  35   __close_nocancel (fd);
  36   if (s < 0)
  37     return 0;
  38
  39   unsigned long int r = 0;
  40   for (ssize_t i = 0; i < s; i++)
  41     {
  42       if (str[i] == '\n')
  43         break;
  44       r *= 10;
  45       r += str[i] - '0';
  46     }
  47   return r;
  48 }
  49
  50 enum malloc_thp_mode_t
  51 __malloc_thp_mode (void)
  52 {
  53   int fd = __open64_nocancel ("/sys/kernel/mm/transparent_hugepage/enabled",
  54                               O_RDONLY);
  55   if (fd == -1)
  56     return malloc_thp_mode_not_supported;
  57
  58   static const char mode_always[]  = "[always] madvise never\n";
  59   static const char mode_madvise[] = "always [madvise] never\n";
  60   static const char mode_never[]   = "always madvise [never]\n";
  61
  62   char str[sizeof(mode_always)];
  63   ssize_t s = __read_nocancel (fd, str, sizeof (str));
  64   __close_nocancel (fd);
  65
  66   if (s == sizeof (mode_always) - 1)
  67     {
  68       if (strcmp (str, mode_always) == 0)
  69         return malloc_thp_mode_always;
  70       else if (strcmp (str, mode_madvise) == 0)
  71         return malloc_thp_mode_madvise;
  72       else if (strcmp (str, mode_never) == 0)
  73         return malloc_thp_mode_never;
  74     }
  75   return malloc_thp_mode_not_supported;
  76 }
  77
  78 static size_t
  79 malloc_default_hugepage_size (void)
  80 {
  81   int fd = __open64_nocancel ("/proc/meminfo", O_RDONLY);
  82   if (fd == -1)
  83     return 0;
  84
  85   size_t hpsize = 0;
  86
  87   char buf[512];
  88   off64_t off = 0;
  89   while (1)
  90     {
  91       ssize_t r = __pread64_nocancel (fd, buf, sizeof (buf) - 1, off);
  92       if (r < 0)
  93         break;
  94       buf[r] = '\0';
  95
  96       /* If the tag is not found, read the last line again.  */
  97       const char *s = strstr (buf, "Hugepagesize:");
  98       if (s == NULL)
  99         {
 100           char *nl = strrchr (buf, '\n');
 101           if (nl == NULL)
 102             break;
 103           off += (nl + 1) - buf;
 104           continue;
 105         }
 106
 107       /* The default huge page size is in the form:
 108          Hugepagesize:       NUMBER kB  */
 109       s += sizeof ("Hugepagesize: ") - 1;
 110       for (int i = 0; (s[i] >= '0' && s[i] <= '9') || s[i] == ' '; i++)
 111         {
 112           if (s[i] == ' ')
 113             continue;
 114           hpsize *= 10;
 115           hpsize += s[i] - '0';
 116         }
 117       hpsize *= 1024;
 118       break;
 119     }
 120
 121   __close_nocancel (fd);
 122
 123   return hpsize;
 124 }
 125
 126 static inline int
 127 hugepage_flags (size_t pagesize)
 128 {
 129   return MAP_HUGETLB | (__builtin_ctzll (pagesize) << MAP_HUGE_SHIFT);
 130 }
 131
 132 void
 133 __malloc_hugepage_config (size_t requested, size_t *pagesize, int *flags)
 134 {
 135   *pagesize = 0;
 136   *flags = 0;
 137
 138   if (requested == 0)
 139     {
 140       *pagesize = malloc_default_hugepage_size ();
 141       if (*pagesize != 0)
 142         *flags = hugepage_flags (*pagesize);
 143       return;
 144     }
 145
 146   /* Each entry represents a supported huge page in the form of:
 147      hugepages-<size>kB.  */
 148   int dirfd = __open64_nocancel ("/sys/kernel/mm/hugepages",
 149                                  O_RDONLY | O_DIRECTORY, 0);
 150   if (dirfd == -1)
 151     return;
 152
 153   char buffer[1024];
 154   while (true)
 155     {
 156 #if !IS_IN(libc)
 157 # define __getdents64 getdents64
 158 #endif
 159       ssize_t ret = __getdents64 (dirfd, buffer, sizeof (buffer));
 160       if (ret == -1)
 161         break;
 162       else if (ret == 0)
 163         break;
 164
 165       bool found = false;
 166       char *begin = buffer, *end = buffer + ret;
 167       while (begin != end)
 168         {
 169           unsigned short int d_reclen;
 170           memcpy (&d_reclen, begin + offsetof (struct dirent64, d_reclen),
 171                   sizeof (d_reclen));
 172           const char *dname = begin + offsetof (struct dirent64, d_name);
 173           begin += d_reclen;
 174
 175           if (dname[0] == '.'
 176               || strncmp (dname, "hugepages-", sizeof ("hugepages-") - 1) != 0)
 177             continue;
 178
 179           size_t hpsize = 0;
 180           const char *sizestr = dname + sizeof ("hugepages-") - 1;
 181           for (int i = 0; sizestr[i] >= '0' && sizestr[i] <= '9'; i++)
 182             {
 183               hpsize *= 10;
 184               hpsize += sizestr[i] - '0';
 185             }
 186           hpsize *= 1024;
 187
 188           if (hpsize == requested)
 189             {
 190               *pagesize = hpsize;
 191               *flags = hugepage_flags (*pagesize);
 192               found = true;
 193               break;
 194             }
 195         }
 196       if (found)
 197         break;
 198     }
 199
 200   __close_nocancel (dirfd);
 201 }