src/liblzma/check/crc32_fast.c

   1 ///////////////////////////////////////////////////////////////////////////////
   2 //
   3 /// \file       crc32.c
   4 /// \brief      CRC32 calculation
   5 //
   6 //  Authors:    Lasse Collin
   7 //              Ilya Kurdyukov
   8 //              Hans Jansen
   9 //
  10 //  This file has been put into the public domain.
  11 //  You can do whatever you want with this file.
  12 //
  13 ///////////////////////////////////////////////////////////////////////////////
  14
  15 #include "check.h"
  16 #include "crc_common.h"
  17
  18 #if defined(CRC_X86_CLMUL)
  19 #       define BUILDING_CRC32_CLMUL
  20 #       include "crc_x86_clmul.h"
  21 #elif defined(CRC32_ARM64)
  22 #       define BUILDING_CRC32_AARCH64
  23 #       include "crc32_aarch64.h"
  24 #endif
  25
  26
  27 #ifdef CRC32_GENERIC
  28
  29 ///////////////////
  30 // Generic CRC32 //
  31 ///////////////////
  32
  33 static uint32_t
  34 crc32_generic(const uint8_t *buf, size_t size, uint32_t crc)
  35 {
  36         crc = ~crc;
  37
  38 #ifdef WORDS_BIGENDIAN
  39         crc = bswap32(crc);
  40 #endif
  41
  42         if (size > 8) {
  43                 // Fix the alignment, if needed. The if statement above
  44                 // ensures that this won't read past the end of buf[].
  45                 while ((uintptr_t)(buf) & 7) {
  46                         crc = lzma_crc32_table[0][*buf++ ^ A(crc)] ^ S8(crc);
  47                         --size;
  48                 }
  49
  50                 // Calculate the position where to stop.
  51                 const uint8_t *const limit = buf + (size & ~(size_t)(7));
  52
  53                 // Calculate how many bytes must be calculated separately
  54                 // before returning the result.
  55                 size &= (size_t)(7);
  56
  57                 // Calculate the CRC32 using the slice-by-eight algorithm.
  58                 while (buf < limit) {
  59                         crc ^= aligned_read32ne(buf);
  60                         buf += 4;
  61
  62                         crc = lzma_crc32_table[7][A(crc)]
  63                             ^ lzma_crc32_table[6][B(crc)]
  64                             ^ lzma_crc32_table[5][C(crc)]
  65                             ^ lzma_crc32_table[4][D(crc)];
  66
  67                         const uint32_t tmp = aligned_read32ne(buf);
  68                         buf += 4;
  69
  70                         // At least with some compilers, it is critical for
  71                         // performance, that the crc variable is XORed
  72                         // between the two table-lookup pairs.
  73                         crc = lzma_crc32_table[3][A(tmp)]
  74                             ^ lzma_crc32_table[2][B(tmp)]
  75                             ^ crc
  76                             ^ lzma_crc32_table[1][C(tmp)]
  77                             ^ lzma_crc32_table[0][D(tmp)];
  78                 }
  79         }
  80
  81         while (size-- != 0)
  82                 crc = lzma_crc32_table[0][*buf++ ^ A(crc)] ^ S8(crc);
  83
  84 #ifdef WORDS_BIGENDIAN
  85         crc = bswap32(crc);
  86 #endif
  87
  88         return ~crc;
  89 }
  90 #endif
  91
  92
  93 #if defined(CRC32_GENERIC) && defined(CRC32_ARCH_OPTIMIZED)
  94
  95 //////////////////////////
  96 // Function dispatching //
  97 //////////////////////////
  98
  99 // If both the generic and arch-optimized implementations are built, then
 100 // the function to use is selected at runtime because the system running
 101 // the binary might not have the arch-specific instruction set extension(s)
 102 // available. The three dispatch methods in order of priority:
 103 //
 104 // 1. Indirect function (ifunc). This method is slightly more efficient
 105 //    than the constructor method because it will change the entry in the
 106 //    Procedure Linkage Table (PLT) for the function either at load time or
 107 //    at the first call. This avoids having to call the function through a
 108 //    function pointer and will treat the function call like a regular call
 109 //    through the PLT. ifuncs are created by using
 110 //    __attribute__((__ifunc__("resolver"))) on a function which has no
 111 //    body. The "resolver" is the name of the function that chooses at
 112 //    runtime which implementation to use.
 113 //
 114 // 2. Constructor. This method uses __attribute__((__constructor__)) to
 115 //    set crc32_func at load time. This avoids extra computation (and any
 116 //    unlikely threading bugs) on the first call to lzma_crc32() to decide
 117 //    which implementation should be used.
 118 //
 119 // 3. First Call Resolution. On the very first call to lzma_crc32(), the
 120 //    call will be directed to crc32_dispatch() instead. This will set the
 121 //    appropriate implementation function and will not be called again.
 122 //    This method does not use any kind of locking but is safe because if
 123 //    multiple threads run the dispatcher simultaneously then they will all
 124 //    set crc32_func to the same value.
 125
 126 typedef uint32_t (*crc32_func_type)(
 127                 const uint8_t *buf, size_t size, uint32_t crc);
 128
 129 // Clang 16.0.0 and older has a bug where it marks the ifunc resolver
 130 // function as unused since it is static and never used outside of
 131 // __attribute__((__ifunc__())).
 132 #if defined(HAVE_FUNC_ATTRIBUTE_IFUNC) && defined(__clang__)
 133 #       pragma GCC diagnostic push
 134 #       pragma GCC diagnostic ignored "-Wunused-function"
 135 #endif
 136
 137 // This resolver is shared between all three dispatch methods. It serves as
 138 // the ifunc resolver if ifunc is supported, otherwise it is called as a
 139 // regular function by the constructor or first call resolution methods.
 140 static crc32_func_type
 141 crc32_resolve(void)
 142 {
 143         return is_arch_extension_supported()
 144                         ? &crc32_arch_optimized : &crc32_generic;
 145 }
 146
 147 #if defined(HAVE_FUNC_ATTRIBUTE_IFUNC) && defined(__clang__)
 148 #       pragma GCC diagnostic pop
 149 #endif
 150
 151 #ifndef HAVE_FUNC_ATTRIBUTE_IFUNC
 152
 153 #ifdef HAVE_FUNC_ATTRIBUTE_CONSTRUCTOR
 154 // Constructor method.
 155 #       define CRC32_SET_FUNC_ATTR __attribute__((__constructor__))
 156 static crc32_func_type crc32_func;
 157 #else
 158 // First Call Resolution method.
 159 #       define CRC32_SET_FUNC_ATTR
 160 static uint32_t crc32_dispatch(const uint8_t *buf, size_t size, uint32_t crc);
 161 static crc32_func_type crc32_func = &crc32_dispatch;
 162 #endif
 163
 164 CRC32_SET_FUNC_ATTR
 165 static void
 166 crc32_set_func(void)
 167 {
 168         crc32_func = crc32_resolve();
 169         return;
 170 }
 171
 172 #ifndef HAVE_FUNC_ATTRIBUTE_CONSTRUCTOR
 173 static uint32_t
 174 crc32_dispatch(const uint8_t *buf, size_t size, uint32_t crc)
 175 {
 176         // When __attribute__((__ifunc__(...))) and
 177         // __attribute__((__constructor__)) isn't supported, set the
 178         // function pointer without any locking. If multiple threads run
 179         // the detection code in parallel, they will all end up setting
 180         // the pointer to the same value. This avoids the use of
 181         // mythread_once() on every call to lzma_crc32() but this likely
 182         // isn't strictly standards compliant. Let's change it if it breaks.
 183         crc32_set_func();
 184         return crc32_func(buf, size, crc);
 185 }
 186
 187 #endif
 188 #endif
 189 #endif
 190
 191
 192 #ifdef CRC_USE_IFUNC
 193 extern LZMA_API(uint32_t)
 194 lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc)
 195                 __attribute__((__ifunc__("crc32_resolve")));
 196 #else
 197 extern LZMA_API(uint32_t)
 198 lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc)
 199 {
 200 #if defined(CRC32_GENERIC) && defined(CRC32_ARCH_OPTIMIZED)
 201         // On x86-64, if CLMUL is available, it is the best for non-tiny
 202         // inputs, being over twice as fast as the generic slice-by-four
 203         // version. However, for size <= 16 it's different. In the extreme
 204         // case of size == 1 the generic version can be five times faster.
 205         // At size >= 8 the CLMUL starts to become reasonable. It
 206         // varies depending on the alignment of buf too.
 207         //
 208         // The above doesn't include the overhead of mythread_once().
 209         // At least on x86-64 GNU/Linux, pthread_once() is very fast but
 210         // it still makes lzma_crc32(buf, 1, crc) 50-100 % slower. When
 211         // size reaches 12-16 bytes the overhead becomes negligible.
 212         //
 213         // So using the generic version for size <= 16 may give better
 214         // performance with tiny inputs but if such inputs happen rarely
 215         // it's not so obvious because then the lookup table of the
 216         // generic version may not be in the processor cache.
 217 #ifdef CRC_USE_GENERIC_FOR_SMALL_INPUTS
 218         if (size <= 16)
 219                 return crc32_generic(buf, size, crc);
 220 #endif
 221
 222 /*
 223 #ifndef HAVE_FUNC_ATTRIBUTE_CONSTRUCTOR
 224         // See crc32_dispatch(). This would be the alternative which uses
 225         // locking and doesn't use crc32_dispatch(). Note that on Windows
 226         // this method needs Vista threads.
 227         mythread_once(crc64_set_func);
 228 #endif
 229 */
 230         return crc32_func(buf, size, crc);
 231
 232 #elif defined(CRC32_ARCH_OPTIMIZED)
 233         return crc32_arch_optimized(buf, size, crc);
 234
 235 #else
 236         return crc32_generic(buf, size, crc);
 237 #endif
 238 }
 239 #endif