support/blob_repeat.c

   1 /* Repeating a memory blob, with alias mapping optimization.
   2    Copyright (C) 2018-2023 Free Software Foundation, Inc.
   3    This file is part of the GNU C Library.
   4
   5    The GNU C Library is free software; you can redistribute it and/or
   6    modify it under the terms of the GNU Lesser General Public
   7    License as published by the Free Software Foundation; either
   8    version 2.1 of the License, or (at your option) any later version.
   9
  10    The GNU C Library is distributed in the hope that it will be useful,
  11    but WITHOUT ANY WARRANTY; without even the implied warranty of
  12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13    Lesser General Public License for more details.
  14
  15    You should have received a copy of the GNU Lesser General Public
  16    License along with the GNU C Library; if not, see
  17    <https://www.gnu.org/licenses/>.  */
  18
  19 #include <errno.h>
  20 #include <fcntl.h>
  21 #include <stdbool.h>
  22 #include <stdlib.h>
  23 #include <string.h>
  24 #include <support/blob_repeat.h>
  25 #include <support/check.h>
  26 #include <support/test-driver.h>
  27 #include <support/support.h>
  28 #include <support/xunistd.h>
  29 #include <sys/mman.h>
  30 #include <unistd.h>
  31 #include <wchar.h>
  32
  33 /* Small allocations should use malloc directly instead of the mmap
  34    optimization because mappings carry a lot of overhead.  */
  35 static const size_t maximum_small_size = 4 * 1024 * 1024;
  36
  37 /* Internal helper for fill.  */
  38 static void
  39 fill0 (char *target, const char *element, size_t element_size,
  40        size_t count)
  41 {
  42   while (count > 0)
  43     {
  44       memcpy (target, element, element_size);
  45       target += element_size;
  46       --count;
  47     }
  48 }
  49
  50 /* Fill the buffer at TARGET with COUNT copies of the ELEMENT_SIZE
  51    bytes starting at ELEMENT.  */
  52 static void
  53 fill (char *target, const char *element, size_t element_size,
  54       size_t count)
  55 {
  56   if (element_size == 0 || count == 0)
  57     return;
  58   else if (element_size == 1)
  59     memset (target, element[0], count);
  60   else if (element_size == sizeof (wchar_t))
  61     {
  62       wchar_t wc;
  63       memcpy (&wc, element, sizeof (wc));
  64       wmemset ((wchar_t *) target, wc, count);
  65     }
  66   else if (element_size < 1024 && count > 4096)
  67     {
  68       /* Use larger copies for really small element sizes.  */
  69       char buffer[8192];
  70       size_t buffer_count = sizeof (buffer) / element_size;
  71       fill0 (buffer, element, element_size, buffer_count);
  72       while (count > 0)
  73         {
  74           size_t copy_count = buffer_count;
  75           if (copy_count > count)
  76             copy_count = count;
  77           size_t copy_bytes = copy_count * element_size;
  78           memcpy (target, buffer, copy_bytes);
  79           target += copy_bytes;
  80           count -= copy_count;
  81         }
  82     }
  83   else
  84     fill0 (target, element, element_size, count);
  85 }
  86
  87 /* Use malloc instead of mmap for small allocations and unusual size
  88    combinations.  */
  89 static struct support_blob_repeat
  90 allocate_malloc (size_t total_size, const void *element, size_t element_size,
  91                  size_t count)
  92 {
  93   void *buffer = malloc (total_size);
  94   if (buffer == NULL)
  95     return (struct support_blob_repeat) { 0 };
  96   fill (buffer, element, element_size, count);
  97   return (struct support_blob_repeat)
  98     {
  99       .start = buffer,
 100       .size = total_size,
 101       .use_malloc = true
 102     };
 103 }
 104
 105 /* Return the least common multiple of PAGE_SIZE and ELEMENT_SIZE,
 106    avoiding overflow.  This assumes that PAGE_SIZE is a power of
 107    two.  */
 108 static size_t
 109 minimum_stride_size (size_t page_size, size_t element_size)
 110 {
 111   TEST_VERIFY_EXIT (page_size > 0);
 112   TEST_VERIFY_EXIT (element_size > 0);
 113
 114   /* Compute the number of trailing zeros common to both sizes.  */
 115   unsigned int common_zeros = __builtin_ctzll (page_size | element_size);
 116
 117   /* In the product, this power of two appears twice, but in the least
 118      common multiple, it appears only once.  Therefore, shift one
 119      factor.  */
 120   size_t multiple;
 121   if (__builtin_mul_overflow (page_size >> common_zeros, element_size,
 122                               &multiple))
 123     return 0;
 124   return multiple;
 125 }
 126
 127 /* Allocations larger than maximum_small_size potentially use mmap
 128    with alias mappings.  If SHARED, the alias mappings are created
 129    using MAP_SHARED instead of MAP_PRIVATE.  */
 130 static struct support_blob_repeat
 131 allocate_big (size_t total_size, const void *element, size_t element_size,
 132               size_t count, bool shared)
 133 {
 134   unsigned long page_size = xsysconf (_SC_PAGESIZE);
 135   size_t stride_size = minimum_stride_size (page_size, element_size);
 136   if (stride_size == 0)
 137     {
 138       errno = EOVERFLOW;
 139       return (struct support_blob_repeat) { 0 };
 140     }
 141
 142   /* Ensure that the stride size is at least maximum_small_size.  This
 143      is necessary to reduce the number of distinct mappings.  */
 144   if (stride_size < maximum_small_size)
 145     stride_size
 146       = ((maximum_small_size + stride_size - 1) / stride_size) * stride_size;
 147
 148   if (stride_size > total_size)
 149     /* The mmap optimization would not save anything.  */
 150     return allocate_malloc (total_size, element, element_size, count);
 151
 152   /* Reserve the memory region.  If we cannot create the mapping,
 153      there is no reason to set up the backing file.  */
 154   void *target = mmap (NULL, total_size, PROT_NONE,
 155                        MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
 156   if (target == MAP_FAILED)
 157     return (struct support_blob_repeat) { 0 };
 158
 159   /* Create the backing file for the repeated mapping.  Call mkstemp
 160      directly to remove the resources backing the temporary file
 161      immediately, once support_blob_repeat_free is called.  Using
 162      create_temp_file would result in a warning during post-test
 163      cleanup.  */
 164   int fd;
 165   {
 166     char *temppath = xasprintf ("%s/support_blob_repeat-XXXXXX", test_dir);
 167     fd = mkstemp (temppath);
 168     if (fd < 0)
 169       FAIL_EXIT1 ("mkstemp (\"%s\"): %m", temppath);
 170     xunlink (temppath);
 171     free (temppath);
 172   }
 173
 174   /* Make sure that there is backing storage, so that the fill
 175      operation will not fault.  */
 176   if (posix_fallocate (fd, 0, stride_size) != 0)
 177     FAIL_EXIT1 ("posix_fallocate (%zu): %m", stride_size);
 178
 179   /* The stride size must still be a multiple of the page size and
 180      element size.  */
 181   TEST_VERIFY_EXIT ((stride_size % page_size) == 0);
 182   TEST_VERIFY_EXIT ((stride_size % element_size) == 0);
 183
 184   /* Fill the backing store.  */
 185   {
 186     void *ptr = mmap (target, stride_size, PROT_READ | PROT_WRITE,
 187                       MAP_FIXED | MAP_FILE | MAP_SHARED, fd, 0);
 188     if (ptr == MAP_FAILED)
 189       {
 190         int saved_errno = errno;
 191         xmunmap (target, total_size);
 192         xclose (fd);
 193         errno = saved_errno;
 194         return (struct support_blob_repeat) { 0 };
 195       }
 196     if (ptr != target)
 197       FAIL_EXIT1 ("mapping of %zu bytes moved from %p to %p",
 198                   stride_size, target, ptr);
 199
 200     /* Write the repeating data.  */
 201     fill (target, element, element_size, stride_size / element_size);
 202
 203     /* Return to a PROT_NONE mapping, just to be on the safe side.  */
 204     ptr = mmap (target, stride_size, PROT_NONE,
 205                 MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE, -1, 0);
 206     if (ptr == MAP_FAILED)
 207       FAIL_EXIT1 ("Failed to reinstate PROT_NONE mapping: %m");
 208     if (ptr != target)
 209       FAIL_EXIT1 ("PROT_NONE mapping of %zu bytes moved from %p to %p",
 210                   stride_size, target, ptr);
 211   }
 212
 213   /* Create the alias mappings.  */
 214   {
 215     size_t remaining_size = total_size;
 216     char *current = target;
 217     int flags = MAP_FIXED | MAP_FILE;
 218     if (shared)
 219       flags |= MAP_SHARED;
 220     else
 221       flags |= MAP_PRIVATE;
 222 #ifdef MAP_NORESERVE
 223     flags |= MAP_NORESERVE;
 224 #endif
 225     while (remaining_size > 0)
 226       {
 227         size_t to_map = stride_size;
 228         if (to_map > remaining_size)
 229           to_map = remaining_size;
 230         void *ptr = mmap (current, to_map, PROT_READ | PROT_WRITE,
 231                           flags, fd, 0);
 232         if (ptr == MAP_FAILED)
 233           {
 234             int saved_errno = errno;
 235             xmunmap (target, total_size);
 236             xclose (fd);
 237             errno = saved_errno;
 238             return (struct support_blob_repeat) { 0 };
 239           }
 240         if (ptr != current)
 241           FAIL_EXIT1 ("MAP_PRIVATE mapping of %zu bytes moved from %p to %p",
 242                       to_map, target, ptr);
 243         remaining_size -= to_map;
 244         current += to_map;
 245       }
 246   }
 247
 248   xclose (fd);
 249
 250   return (struct support_blob_repeat)
 251     {
 252       .start = target,
 253       .size = total_size,
 254       .use_malloc = false
 255     };
 256 }
 257
 258 struct support_blob_repeat
 259 repeat_allocate (const void *element, size_t element_size,
 260                  size_t count, bool shared)
 261 {
 262   size_t total_size;
 263   if (__builtin_mul_overflow (element_size, count, &total_size))
 264     {
 265       errno = EOVERFLOW;
 266       return (struct support_blob_repeat) { 0 };
 267     }
 268   if (total_size <= maximum_small_size)
 269     return allocate_malloc (total_size, element, element_size, count);
 270   else
 271     return allocate_big (total_size, element, element_size, count, shared);
 272 }
 273
 274 struct support_blob_repeat
 275 support_blob_repeat_allocate (const void *element, size_t element_size,
 276                               size_t count)
 277 {
 278   return repeat_allocate (element, element_size, count, false);
 279 }
 280
 281 struct support_blob_repeat
 282 support_blob_repeat_allocate_shared (const void *element, size_t element_size,
 283                                      size_t count)
 284 {
 285   return repeat_allocate (element, element_size, count, true);
 286 }
 287
 288 void
 289 support_blob_repeat_free (struct support_blob_repeat *blob)
 290 {
 291   if (blob->size > 0)
 292     {
 293       int saved_errno = errno;
 294       if (blob->use_malloc)
 295         free (blob->start);
 296       else
 297         xmunmap (blob->start, blob->size);
 298       errno = saved_errno;
 299     }
 300   *blob = (struct support_blob_repeat) { 0 };
 301 }