gcc/lto-compress.cc

   1 /* LTO IL compression streams.
   2
   3    Copyright (C) 2009-2023 Free Software Foundation, Inc.
   4    Contributed by Simon Baldwin <simonb@google.com>
   5
   6 This file is part of GCC.
   7
   8 GCC is free software; you can redistribute it and/or modify it
   9 under the terms of the GNU General Public License as published by
  10 the Free Software Foundation; either version 3, or (at your option)
  11 any later version.
  12
  13 GCC is distributed in the hope that it will be useful, but WITHOUT
  14 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
  15 or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public
  16 License for more details.
  17
  18 You should have received a copy of the GNU General Public License
  19 along with GCC; see the file COPYING3.  If not see
  20 <http://www.gnu.org/licenses/>.  */
  21
  22 #include "config.h"
  23 #include "system.h"
  24 #include "coretypes.h"
  25 #include "backend.h"
  26 #include "tree.h"
  27 #include "gimple.h"
  28 #include "cgraph.h"
  29 #include "lto-streamer.h"
  30 /* zlib.h includes other system headers.  Those headers may test feature
  31    test macros.  config.h may define feature test macros.  For this reason,
  32    zlib.h needs to be included after, rather than before, config.h and
  33    system.h.  */
  34 #include <zlib.h>
  35 #include "lto-compress.h"
  36 #include "timevar.h"
  37
  38 #ifdef HAVE_ZSTD_H
  39 #include <zstd.h>
  40 #endif
  41
  42 /* Compression stream structure, holds the flush callback and opaque token,
  43    the buffered data, and a note of whether compressing or uncompressing.  */
  44
  45 struct lto_compression_stream
  46 {
  47   void (*callback) (const char *, unsigned, void *);
  48   void *opaque;
  49   char *buffer;
  50   size_t bytes;
  51   size_t allocation;
  52   bool is_compression;
  53 };
  54
  55 /* Overall compression constants for zlib.  */
  56
  57 static const size_t Z_BUFFER_LENGTH = 4096;
  58 static const size_t MIN_STREAM_ALLOCATION = 1024;
  59
  60 /* For zlib, allocate SIZE count of ITEMS and return the address, OPAQUE
  61    is unused.  */
  62
  63 static void *
  64 lto_zalloc (void *opaque, unsigned items, unsigned size)
  65 {
  66   gcc_assert (opaque == Z_NULL);
  67   return xmalloc (items * size);
  68 }
  69
  70 /* For zlib, free memory at ADDRESS, OPAQUE is unused.  */
  71
  72 static void
  73 lto_zfree (void *opaque, void *address)
  74 {
  75   gcc_assert (opaque == Z_NULL);
  76   free (address);
  77 }
  78
  79 /* Return a zlib compression level that zlib will not reject.  Normalizes
  80    the compression level from the command line flag, clamping non-default
  81    values to the appropriate end of their valid range.  */
  82
  83 static int
  84 lto_normalized_zlib_level (void)
  85 {
  86   int level = flag_lto_compression_level;
  87
  88   if (level != Z_DEFAULT_COMPRESSION)
  89     {
  90       if (level < Z_NO_COMPRESSION)
  91         level = Z_NO_COMPRESSION;
  92       else if (level > Z_BEST_COMPRESSION)
  93         level = Z_BEST_COMPRESSION;
  94     }
  95
  96   return level;
  97 }
  98
  99 /* Free the buffer and memory associated with STREAM.  */
 100
 101 static void
 102 lto_destroy_compression_stream (struct lto_compression_stream *stream)
 103 {
 104   free (stream->buffer);
 105   free (stream);
 106 }
 107
 108 #ifdef HAVE_ZSTD_H
 109 /* Return a zstd compression level that zstd will not reject.  Normalizes
 110    the compression level from the command line flag, clamping non-default
 111    values to the appropriate end of their valid range.  */
 112
 113 static int
 114 lto_normalized_zstd_level (void)
 115 {
 116   int level = flag_lto_compression_level;
 117
 118   if (level < 0)
 119     level = 0;
 120   else if (level > ZSTD_maxCLevel ())
 121     level = ZSTD_maxCLevel ();
 122
 123   return level;
 124 }
 125
 126 /* Compress STREAM using ZSTD algorithm.  */
 127
 128 static void
 129 lto_compression_zstd (struct lto_compression_stream *stream)
 130 {
 131   unsigned char *cursor = (unsigned char *) stream->buffer;
 132   size_t size = stream->bytes;
 133
 134   timevar_push (TV_IPA_LTO_COMPRESS);
 135   size_t const outbuf_length = ZSTD_compressBound (size);
 136   char *outbuf = (char *) xmalloc (outbuf_length);
 137
 138   size_t const csize = ZSTD_compress (outbuf, outbuf_length, cursor, size,
 139                                       lto_normalized_zstd_level ());
 140
 141   if (ZSTD_isError (csize))
 142     internal_error ("compressed stream: %s", ZSTD_getErrorName (csize));
 143
 144   lto_stats.num_compressed_il_bytes += csize;
 145   stream->callback (outbuf, csize, NULL);
 146
 147   lto_destroy_compression_stream (stream);
 148   free (outbuf);
 149   timevar_pop (TV_IPA_LTO_COMPRESS);
 150 }
 151
 152 /* Uncompress STREAM using ZSTD algorithm.  */
 153
 154 static void
 155 lto_uncompression_zstd (struct lto_compression_stream *stream)
 156 {
 157   unsigned char *cursor = (unsigned char *) stream->buffer;
 158   size_t size = stream->bytes;
 159
 160   timevar_push (TV_IPA_LTO_DECOMPRESS);
 161   unsigned long long const rsize = ZSTD_getFrameContentSize (cursor, size);
 162   if (rsize == ZSTD_CONTENTSIZE_ERROR)
 163     internal_error ("original not compressed with zstd");
 164   else if (rsize == ZSTD_CONTENTSIZE_UNKNOWN)
 165     internal_error ("original size unknown");
 166
 167   char *outbuf = (char *) xmalloc (rsize);
 168   size_t const dsize = ZSTD_decompress (outbuf, rsize, cursor, size);
 169
 170   if (ZSTD_isError (dsize))
 171     internal_error ("decompressed stream: %s", ZSTD_getErrorName (dsize));
 172
 173   lto_stats.num_uncompressed_il_bytes += dsize;
 174   stream->callback (outbuf, dsize, stream->opaque);
 175
 176   lto_destroy_compression_stream (stream);
 177   free (outbuf);
 178   timevar_pop (TV_IPA_LTO_DECOMPRESS);
 179 }
 180
 181 #endif
 182
 183 /* Create a new compression stream, with CALLBACK flush function passed
 184    OPAQUE token, IS_COMPRESSION indicates if compressing or uncompressing.  */
 185
 186 static struct lto_compression_stream *
 187 lto_new_compression_stream (void (*callback) (const char *, unsigned, void *),
 188                             void *opaque, bool is_compression)
 189 {
 190   struct lto_compression_stream *stream
 191     = (struct lto_compression_stream *) xmalloc (sizeof (*stream));
 192
 193   memset (stream, 0, sizeof (*stream));
 194   stream->callback = callback;
 195   stream->opaque = opaque;
 196   stream->is_compression = is_compression;
 197
 198   return stream;
 199 }
 200
 201 /* Append NUM_CHARS from address BASE to STREAM.  */
 202
 203 static void
 204 lto_append_to_compression_stream (struct lto_compression_stream *stream,
 205                                   const char *base, size_t num_chars)
 206 {
 207   size_t required = stream->bytes + num_chars;
 208
 209   if (stream->allocation < required)
 210     {
 211       if (stream->allocation == 0)
 212         stream->allocation = MIN_STREAM_ALLOCATION;
 213       while (stream->allocation < required)
 214         stream->allocation *= 2;
 215
 216       stream->buffer = (char *) xrealloc (stream->buffer, stream->allocation);
 217     }
 218
 219   memcpy (stream->buffer + stream->bytes, base, num_chars);
 220   stream->bytes += num_chars;
 221 }
 222
 223 /* Return a new compression stream, with CALLBACK flush function passed
 224    OPAQUE token.  */
 225
 226 struct lto_compression_stream *
 227 lto_start_compression (void (*callback) (const char *, unsigned, void *),
 228                        void *opaque)
 229 {
 230   return lto_new_compression_stream (callback, opaque, true);
 231 }
 232
 233 /* Append NUM_CHARS from address BASE to STREAM.  */
 234
 235 void
 236 lto_compress_block (struct lto_compression_stream *stream,
 237                     const char *base, size_t num_chars)
 238 {
 239   gcc_assert (stream->is_compression);
 240
 241   lto_append_to_compression_stream (stream, base, num_chars);
 242   lto_stats.num_output_il_bytes += num_chars;
 243 }
 244
 245 static void ATTRIBUTE_UNUSED
 246 lto_compression_zlib (struct lto_compression_stream *stream)
 247 {
 248   unsigned char *cursor = (unsigned char *) stream->buffer;
 249   size_t remaining = stream->bytes;
 250   const size_t outbuf_length = Z_BUFFER_LENGTH;
 251   unsigned char *outbuf = (unsigned char *) xmalloc (outbuf_length);
 252   z_stream out_stream;
 253   int status;
 254
 255   gcc_assert (stream->is_compression);
 256
 257   timevar_push (TV_IPA_LTO_COMPRESS);
 258
 259   out_stream.next_out = outbuf;
 260   out_stream.avail_out = outbuf_length;
 261   out_stream.next_in = cursor;
 262   out_stream.avail_in = remaining;
 263   out_stream.zalloc = lto_zalloc;
 264   out_stream.zfree = lto_zfree;
 265   out_stream.opaque = Z_NULL;
 266
 267   status = deflateInit (&out_stream, lto_normalized_zlib_level ());
 268   if (status != Z_OK)
 269     internal_error ("compressed stream: %s", zError (status));
 270
 271   do
 272     {
 273       size_t in_bytes, out_bytes;
 274
 275       status = deflate (&out_stream, Z_FINISH);
 276       if (status != Z_OK && status != Z_STREAM_END)
 277         internal_error ("compressed stream: %s", zError (status));
 278
 279       in_bytes = remaining - out_stream.avail_in;
 280       out_bytes = outbuf_length - out_stream.avail_out;
 281
 282       stream->callback ((const char *) outbuf, out_bytes, stream->opaque);
 283       lto_stats.num_compressed_il_bytes += out_bytes;
 284
 285       cursor += in_bytes;
 286       remaining -= in_bytes;
 287
 288       out_stream.next_out = outbuf;
 289       out_stream.avail_out = outbuf_length;
 290       out_stream.next_in = cursor;
 291       out_stream.avail_in = remaining;
 292     }
 293   while (status != Z_STREAM_END);
 294
 295   status = deflateEnd (&out_stream);
 296   if (status != Z_OK)
 297     internal_error ("compressed stream: %s", zError (status));
 298
 299   lto_destroy_compression_stream (stream);
 300   free (outbuf);
 301   timevar_pop (TV_IPA_LTO_COMPRESS);
 302 }
 303
 304 void
 305 lto_end_compression (struct lto_compression_stream *stream)
 306 {
 307 #ifdef HAVE_ZSTD_H
 308   lto_compression_zstd (stream);
 309 #else
 310   lto_compression_zlib (stream);
 311 #endif
 312 }
 313
 314 /* Return a new uncompression stream, with CALLBACK flush function passed
 315    OPAQUE token.  */
 316
 317 struct lto_compression_stream *
 318 lto_start_uncompression (void (*callback) (const char *, unsigned, void *),
 319                          void *opaque)
 320 {
 321   return lto_new_compression_stream (callback, opaque, false);
 322 }
 323
 324 /* Append NUM_CHARS from address BASE to STREAM.  */
 325
 326 void
 327 lto_uncompress_block (struct lto_compression_stream *stream,
 328                       const char *base, size_t num_chars)
 329 {
 330   gcc_assert (!stream->is_compression);
 331
 332   lto_append_to_compression_stream (stream, base, num_chars);
 333   lto_stats.num_input_il_bytes += num_chars;
 334 }
 335
 336 static void
 337 lto_uncompression_zlib (struct lto_compression_stream *stream)
 338 {
 339   unsigned char *cursor = (unsigned char *) stream->buffer;
 340   size_t remaining = stream->bytes;
 341   const size_t outbuf_length = Z_BUFFER_LENGTH;
 342   unsigned char *outbuf = (unsigned char *) xmalloc (outbuf_length);
 343
 344   gcc_assert (!stream->is_compression);
 345   timevar_push (TV_IPA_LTO_DECOMPRESS);
 346
 347   while (remaining > 0)
 348     {
 349       z_stream in_stream;
 350       size_t out_bytes;
 351       int status;
 352
 353       in_stream.next_out = outbuf;
 354       in_stream.avail_out = outbuf_length;
 355       in_stream.next_in = cursor;
 356       in_stream.avail_in = remaining;
 357       in_stream.zalloc = lto_zalloc;
 358       in_stream.zfree = lto_zfree;
 359       in_stream.opaque = Z_NULL;
 360
 361       status = inflateInit (&in_stream);
 362       if (status != Z_OK)
 363         internal_error ("compressed stream: %s", zError (status));
 364
 365       do
 366         {
 367           size_t in_bytes;
 368
 369           status = inflate (&in_stream, Z_SYNC_FLUSH);
 370           if (status != Z_OK && status != Z_STREAM_END)
 371             internal_error ("compressed stream: %s", zError (status));
 372
 373           in_bytes = remaining - in_stream.avail_in;
 374           out_bytes = outbuf_length - in_stream.avail_out;
 375
 376           stream->callback ((const char *) outbuf, out_bytes, stream->opaque);
 377           lto_stats.num_uncompressed_il_bytes += out_bytes;
 378
 379           cursor += in_bytes;
 380           remaining -= in_bytes;
 381
 382           in_stream.next_out = outbuf;
 383           in_stream.avail_out = outbuf_length;
 384           in_stream.next_in = cursor;
 385           in_stream.avail_in = remaining;
 386         }
 387       while (!(status == Z_STREAM_END && out_bytes == 0));
 388
 389       status = inflateEnd (&in_stream);
 390       if (status != Z_OK)
 391         internal_error ("compressed stream: %s", zError (status));
 392     }
 393
 394   lto_destroy_compression_stream (stream);
 395   free (outbuf);
 396   timevar_pop (TV_IPA_LTO_DECOMPRESS);
 397 }
 398
 399 void
 400 lto_end_uncompression (struct lto_compression_stream *stream,
 401                        lto_compression compression)
 402 {
 403 #ifdef HAVE_ZSTD_H
 404   if (compression == ZSTD)
 405     {
 406       lto_uncompression_zstd (stream);
 407       return;
 408     }
 409 #endif
 410   if (compression == ZSTD)
 411     internal_error ("compiler does not support ZSTD LTO compression");
 412
 413   lto_uncompression_zlib (stream);
 414 }