stdlib/qsort.c

   1 /* Copyright (C) 1991-2023 Free Software Foundation, Inc.
   2    This file is part of the GNU C Library.
   3
   4    The GNU C Library is free software; you can redistribute it and/or
   5    modify it under the terms of the GNU Lesser General Public
   6    License as published by the Free Software Foundation; either
   7    version 2.1 of the License, or (at your option) any later version.
   8
   9    The GNU C Library is distributed in the hope that it will be useful,
  10    but WITHOUT ANY WARRANTY; without even the implied warranty of
  11    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  12    Lesser General Public License for more details.
  13
  14    You should have received a copy of the GNU Lesser General Public
  15    License along with the GNU C Library; if not, see
  16    <https://www.gnu.org/licenses/>.  */
  17
  18 /* If you consider tuning this algorithm, you should consult first:
  19    Engineering a sort function; Jon Bentley and M. Douglas McIlroy;
  20    Software - Practice and Experience; Vol. 23 (11), 1249-1265, 1993.  */
  21
  22 #include <alloca.h>
  23 #include <limits.h>
  24 #include <memswap.h>
  25 #include <stdlib.h>
  26 #include <string.h>
  27 #include <stdbool.h>
  28
  29 /* Swap SIZE bytes between addresses A and B.  These helpers are provided
  30    along the generic one as an optimization.  */
  31
  32 enum swap_type_t
  33   {
  34     SWAP_WORDS_64,
  35     SWAP_WORDS_32,
  36     SWAP_BYTES
  37   };
  38
  39 /* If this function returns true, elements can be safely copied using word
  40    loads and stores.  Otherwise, it might not be safe.  BASE (as an integer)
  41    must be a multiple of the word alignment.  SIZE must be a multiple of
  42    WORDSIZE.  Since WORDSIZE must be a multiple of the word alignment, and
  43    WORDSIZE is a power of two on all supported platforms, this function for
  44    speed merely checks that BASE and SIZE are both multiples of the word
  45    size.  */
  46 static inline bool
  47 is_aligned (const void *base, size_t size, size_t wordsize)
  48 {
  49   return (((uintptr_t) base | size) & (wordsize - 1)) == 0;
  50 }
  51
  52 static inline void
  53 swap_words_64 (void * restrict a, void * restrict b, size_t n)
  54 {
  55   typedef uint64_t __attribute__ ((__may_alias__)) u64_alias_t;
  56   do
  57    {
  58      n -= 8;
  59      u64_alias_t t = *(u64_alias_t *)(a + n);
  60      *(u64_alias_t *)(a + n) = *(u64_alias_t *)(b + n);
  61      *(u64_alias_t *)(b + n) = t;
  62    } while (n);
  63 }
  64
  65 static inline void
  66 swap_words_32 (void * restrict a, void * restrict b, size_t n)
  67 {
  68   typedef uint32_t __attribute__ ((__may_alias__)) u32_alias_t;
  69   do
  70    {
  71      n -= 4;
  72      u32_alias_t t = *(u32_alias_t *)(a + n);
  73      *(u32_alias_t *)(a + n) = *(u32_alias_t *)(b + n);
  74      *(u32_alias_t *)(b + n) = t;
  75    } while (n);
  76 }
  77
  78 /* Replace the indirect call with a serie of if statements.  It should help
  79    the branch predictor.  */
  80 static void
  81 do_swap (void * restrict a, void * restrict b, size_t size,
  82          enum swap_type_t swap_type)
  83 {
  84   if (swap_type == SWAP_WORDS_64)
  85     swap_words_64 (a, b, size);
  86   else if (swap_type == SWAP_WORDS_32)
  87     swap_words_32 (a, b, size);
  88   else
  89     __memswap (a, b, size);
  90 }
  91
  92 /* Discontinue quicksort algorithm when partition gets below this size.
  93    This particular magic number was chosen to work best on a Sun 4/260. */
  94 #define MAX_THRESH 4
  95
  96 /* Stack node declarations used to store unfulfilled partition obligations. */
  97 typedef struct
  98   {
  99     char *lo;
 100     char *hi;
 101   } stack_node;
 102
 103 /* The next 4 #defines implement a very fast in-line stack abstraction. */
 104 /* The stack needs log (total_elements) entries (we could even subtract
 105    log(MAX_THRESH)).  Since total_elements has type size_t, we get as
 106    upper bound for log (total_elements):
 107    bits per byte (CHAR_BIT) * sizeof(size_t).  */
 108 #define STACK_SIZE      (CHAR_BIT * sizeof (size_t))
 109 #define PUSH(low, high) ((void) ((top->lo = (low)), (top->hi = (high)), ++top))
 110 #define POP(low, high)  ((void) (--top, (low = top->lo), (high = top->hi)))
 111 #define STACK_NOT_EMPTY (stack < top)
 112
 113
 114 static inline void
 115 insertion_sort_qsort_partitions (void *const pbase, size_t total_elems,
 116                                  size_t size, enum swap_type_t swap_type,
 117                                  __compar_d_fn_t cmp, void *arg)
 118 {
 119   char *base_ptr = (char *) pbase;
 120   char *const end_ptr = &base_ptr[size * (total_elems - 1)];
 121   char *tmp_ptr = base_ptr;
 122 #define min(x, y) ((x) < (y) ? (x) : (y))
 123   const size_t max_thresh = MAX_THRESH * size;
 124   char *thresh = min(end_ptr, base_ptr + max_thresh);
 125   char *run_ptr;
 126
 127   /* Find smallest element in first threshold and place it at the
 128      array's beginning.  This is the smallest array element,
 129      and the operation speeds up insertion sort's inner loop. */
 130
 131   for (run_ptr = tmp_ptr + size; run_ptr <= thresh; run_ptr += size)
 132     if (cmp (run_ptr, tmp_ptr, arg) < 0)
 133       tmp_ptr = run_ptr;
 134
 135   if (tmp_ptr != base_ptr)
 136     do_swap (tmp_ptr, base_ptr, size, swap_type);
 137
 138   /* Insertion sort, running from left-hand-side up to right-hand-side.  */
 139
 140   run_ptr = base_ptr + size;
 141   while ((run_ptr += size) <= end_ptr)
 142     {
 143       tmp_ptr = run_ptr - size;
 144       while (cmp (run_ptr, tmp_ptr, arg) < 0)
 145         tmp_ptr -= size;
 146
 147       tmp_ptr += size;
 148       if (tmp_ptr != run_ptr)
 149         {
 150           char *trav;
 151
 152           trav = run_ptr + size;
 153           while (--trav >= run_ptr)
 154             {
 155               char c = *trav;
 156               char *hi, *lo;
 157
 158               for (hi = lo = trav; (lo -= size) >= tmp_ptr; hi = lo)
 159                 *hi = *lo;
 160               *hi = c;
 161             }
 162         }
 163     }
 164 }
 165
 166 /* Order size using quicksort.  This implementation incorporates
 167    four optimizations discussed in Sedgewick:
 168
 169    1. Non-recursive, using an explicit stack of pointer that store the
 170       next array partition to sort.  To save time, this maximum amount
 171       of space required to store an array of SIZE_MAX is allocated on the
 172       stack.  Assuming a 32-bit (64 bit) integer for size_t, this needs
 173       only 32 * sizeof(stack_node) == 256 bytes (for 64 bit: 1024 bytes).
 174       Pretty cheap, actually.
 175
 176    2. Chose the pivot element using a median-of-three decision tree.
 177       This reduces the probability of selecting a bad pivot value and
 178       eliminates certain extraneous comparisons.
 179
 180    3. Only quicksorts TOTAL_ELEMS / MAX_THRESH partitions, leaving
 181       insertion sort to order the MAX_THRESH items within each partition.
 182       This is a big win, since insertion sort is faster for small, mostly
 183       sorted array segments.
 184
 185    4. The larger of the two sub-partitions is always pushed onto the
 186       stack first, with the algorithm then concentrating on the
 187       smaller partition.  This *guarantees* no more than log (total_elems)
 188       stack size is needed (actually O(1) in this case)!  */
 189
 190 void
 191 _quicksort (void *const pbase, size_t total_elems, size_t size,
 192             __compar_d_fn_t cmp, void *arg)
 193 {
 194   char *base_ptr = (char *) pbase;
 195
 196   const size_t max_thresh = MAX_THRESH * size;
 197
 198   if (total_elems == 0)
 199     /* Avoid lossage with unsigned arithmetic below.  */
 200     return;
 201
 202   enum swap_type_t swap_type;
 203   if (is_aligned (pbase, size, 8))
 204     swap_type = SWAP_WORDS_64;
 205   else if (is_aligned (pbase, size, 4))
 206     swap_type = SWAP_WORDS_32;
 207   else
 208     swap_type = SWAP_BYTES;
 209
 210   if (total_elems > MAX_THRESH)
 211     {
 212       char *lo = base_ptr;
 213       char *hi = &lo[size * (total_elems - 1)];
 214       stack_node stack[STACK_SIZE];
 215       stack_node *top = stack;
 216
 217       PUSH (NULL, NULL);
 218
 219       while (STACK_NOT_EMPTY)
 220         {
 221           char *left_ptr;
 222           char *right_ptr;
 223
 224           /* Select median value from among LO, MID, and HI. Rearrange
 225              LO and HI so the three values are sorted. This lowers the
 226              probability of picking a pathological pivot value and
 227              skips a comparison for both the LEFT_PTR and RIGHT_PTR in
 228              the while loops. */
 229
 230           char *mid = lo + size * ((hi - lo) / size >> 1);
 231
 232           if ((*cmp) ((void *) mid, (void *) lo, arg) < 0)
 233             do_swap (mid, lo, size, swap_type);
 234           if ((*cmp) ((void *) hi, (void *) mid, arg) < 0)
 235             do_swap (mid, hi, size, swap_type);
 236           else
 237             goto jump_over;
 238           if ((*cmp) ((void *) mid, (void *) lo, arg) < 0)
 239             do_swap (mid, lo, size, swap_type);
 240         jump_over:;
 241
 242           left_ptr  = lo + size;
 243           right_ptr = hi - size;
 244
 245           /* Here's the famous ``collapse the walls'' section of quicksort.
 246              Gotta like those tight inner loops!  They are the main reason
 247              that this algorithm runs much faster than others. */
 248           do
 249             {
 250               while ((*cmp) ((void *) left_ptr, (void *) mid, arg) < 0)
 251                 left_ptr += size;
 252
 253               while ((*cmp) ((void *) mid, (void *) right_ptr, arg) < 0)
 254                 right_ptr -= size;
 255
 256               if (left_ptr < right_ptr)
 257                 {
 258                   do_swap (left_ptr, right_ptr, size, swap_type);
 259                   if (mid == left_ptr)
 260                     mid = right_ptr;
 261                   else if (mid == right_ptr)
 262                     mid = left_ptr;
 263                   left_ptr += size;
 264                   right_ptr -= size;
 265                 }
 266               else if (left_ptr == right_ptr)
 267                 {
 268                   left_ptr += size;
 269                   right_ptr -= size;
 270                   break;
 271                 }
 272             }
 273           while (left_ptr <= right_ptr);
 274
 275           /* Set up pointers for next iteration.  First determine whether
 276              left and right partitions are below the threshold size.  If so,
 277              ignore one or both.  Otherwise, push the larger partition's
 278              bounds on the stack and continue sorting the smaller one. */
 279
 280           if ((size_t) (right_ptr - lo) <= max_thresh)
 281             {
 282               if ((size_t) (hi - left_ptr) <= max_thresh)
 283                 /* Ignore both small partitions. */
 284                 POP (lo, hi);
 285               else
 286                 /* Ignore small left partition. */
 287                 lo = left_ptr;
 288             }
 289           else if ((size_t) (hi - left_ptr) <= max_thresh)
 290             /* Ignore small right partition. */
 291             hi = right_ptr;
 292           else if ((right_ptr - lo) > (hi - left_ptr))
 293             {
 294               /* Push larger left partition indices. */
 295               PUSH (lo, right_ptr);
 296               lo = left_ptr;
 297             }
 298           else
 299             {
 300               /* Push larger right partition indices. */
 301               PUSH (left_ptr, hi);
 302               hi = right_ptr;
 303             }
 304         }
 305     }
 306
 307   /* Once the BASE_PTR array is partially sorted by quicksort the rest
 308      is completely sorted using insertion sort, since this is efficient
 309      for partitions below MAX_THRESH size. BASE_PTR points to the beginning
 310      of the array to sort, and END_PTR points at the very last element in
 311      the array (*not* one beyond it!). */
 312   insertion_sort_qsort_partitions (pbase, total_elems, size, swap_type, cmp,
 313                                    arg);
 314 }