third_party/jpeg-xl/lib/jpegli/encode.cc

   1 // Copyright (c) the JPEG XL Project Authors. All rights reserved.
   2 //
   3 // Use of this source code is governed by a BSD-style
   4 // license that can be found in the LICENSE file.
   5
   6 #include "lib/jpegli/encode.h"
   7
   8 #include <jxl/types.h>
   9
  10 #include <cmath>
  11 #include <initializer_list>
  12 #include <vector>
  13
  14 #include "lib/jpegli/adaptive_quantization.h"
  15 #include "lib/jpegli/bit_writer.h"
  16 #include "lib/jpegli/bitstream.h"
  17 #include "lib/jpegli/color_transform.h"
  18 #include "lib/jpegli/downsample.h"
  19 #include "lib/jpegli/encode_finish.h"
  20 #include "lib/jpegli/encode_internal.h"
  21 #include "lib/jpegli/encode_streaming.h"
  22 #include "lib/jpegli/entropy_coding.h"
  23 #include "lib/jpegli/error.h"
  24 #include "lib/jpegli/huffman.h"
  25 #include "lib/jpegli/input.h"
  26 #include "lib/jpegli/memory_manager.h"
  27 #include "lib/jpegli/quant.h"
  28
  29 namespace jpegli {
  30
  31 constexpr size_t kMaxBytesInMarker = 65533;
  32
  33 void CheckState(j_compress_ptr cinfo, int state) {
  34   if (cinfo->global_state != state) {
  35     JPEGLI_ERROR("Unexpected global state %d [expected %d]",
  36                  cinfo->global_state, state);
  37   }
  38 }
  39
  40 void CheckState(j_compress_ptr cinfo, int state1, int state2) {
  41   if (cinfo->global_state != state1 && cinfo->global_state != state2) {
  42     JPEGLI_ERROR("Unexpected global state %d [expected %d or %d]",
  43                  cinfo->global_state, state1, state2);
  44   }
  45 }
  46
  47 //
  48 // Parameter setup
  49 //
  50
  51 // Initialize cinfo fields that are not dependent on input image. This is shared
  52 // between jpegli_CreateCompress() and jpegli_set_defaults()
  53 void InitializeCompressParams(j_compress_ptr cinfo) {
  54   cinfo->data_precision = 8;
  55   cinfo->num_scans = 0;
  56   cinfo->scan_info = nullptr;
  57   cinfo->raw_data_in = FALSE;
  58   cinfo->arith_code = FALSE;
  59   cinfo->optimize_coding = FALSE;
  60   cinfo->CCIR601_sampling = FALSE;
  61   cinfo->smoothing_factor = 0;
  62   cinfo->dct_method = JDCT_FLOAT;
  63   cinfo->restart_interval = 0;
  64   cinfo->restart_in_rows = 0;
  65   cinfo->write_JFIF_header = FALSE;
  66   cinfo->JFIF_major_version = 1;
  67   cinfo->JFIF_minor_version = 1;
  68   cinfo->density_unit = 0;
  69   cinfo->X_density = 1;
  70   cinfo->Y_density = 1;
  71 #if JPEG_LIB_VERSION >= 70
  72   cinfo->scale_num = 1;
  73   cinfo->scale_denom = 1;
  74   cinfo->do_fancy_downsampling = FALSE;
  75   cinfo->min_DCT_h_scaled_size = DCTSIZE;
  76   cinfo->min_DCT_v_scaled_size = DCTSIZE;
  77 #endif
  78   cinfo->master->psnr_target = 0.0f;
  79   cinfo->master->psnr_tolerance = 0.01f;
  80   cinfo->master->min_distance = 0.1f;
  81   cinfo->master->max_distance = 25.0f;
  82 }
  83
  84 float LinearQualityToDistance(int scale_factor) {
  85   scale_factor = std::min(5000, std::max(0, scale_factor));
  86   int quality =
  87       scale_factor < 100 ? 100 - scale_factor / 2 : 5000 / scale_factor;
  88   return jpegli_quality_to_distance(quality);
  89 }
  90
  91 template <typename T>
  92 void SetSentTableFlag(T** table_ptrs, size_t num, boolean val) {
  93   for (size_t i = 0; i < num; ++i) {
  94     if (table_ptrs[i]) table_ptrs[i]->sent_table = val;
  95   }
  96 }
  97
  98 //
  99 // Compressor initialization
 100 //
 101
 102 struct ProgressiveScan {
 103   int Ss, Se, Ah, Al;
 104   bool interleaved;
 105 };
 106
 107 void SetDefaultScanScript(j_compress_ptr cinfo) {
 108   int level = cinfo->master->progressive_level;
 109   std::vector<ProgressiveScan> progressive_mode;
 110   bool interleave_dc =
 111       (cinfo->max_h_samp_factor == 1 && cinfo->max_v_samp_factor == 1);
 112   if (level == 0) {
 113     progressive_mode.push_back({0, 63, 0, 0, true});
 114   } else if (level == 1) {
 115     progressive_mode.push_back({0, 0, 0, 0, interleave_dc});
 116     progressive_mode.push_back({1, 63, 0, 1, false});
 117     progressive_mode.push_back({1, 63, 1, 0, false});
 118   } else {
 119     progressive_mode.push_back({0, 0, 0, 0, interleave_dc});
 120     progressive_mode.push_back({1, 2, 0, 0, false});
 121     progressive_mode.push_back({3, 63, 0, 2, false});
 122     progressive_mode.push_back({3, 63, 2, 1, false});
 123     progressive_mode.push_back({3, 63, 1, 0, false});
 124   }
 125
 126   cinfo->script_space_size = 0;
 127   for (const auto& scan : progressive_mode) {
 128     int comps = scan.interleaved ? MAX_COMPS_IN_SCAN : 1;
 129     cinfo->script_space_size += DivCeil(cinfo->num_components, comps);
 130   }
 131   cinfo->script_space =
 132       Allocate<jpeg_scan_info>(cinfo, cinfo->script_space_size);
 133
 134   jpeg_scan_info* next_scan = cinfo->script_space;
 135   for (const auto& scan : progressive_mode) {
 136     int comps = scan.interleaved ? MAX_COMPS_IN_SCAN : 1;
 137     for (int c = 0; c < cinfo->num_components; c += comps) {
 138       next_scan->Ss = scan.Ss;
 139       next_scan->Se = scan.Se;
 140       next_scan->Ah = scan.Ah;
 141       next_scan->Al = scan.Al;
 142       next_scan->comps_in_scan = std::min(comps, cinfo->num_components - c);
 143       for (int j = 0; j < next_scan->comps_in_scan; ++j) {
 144         next_scan->component_index[j] = c + j;
 145       }
 146       ++next_scan;
 147     }
 148   }
 149   JPEGLI_CHECK(next_scan - cinfo->script_space == cinfo->script_space_size);
 150   cinfo->scan_info = cinfo->script_space;
 151   cinfo->num_scans = cinfo->script_space_size;
 152 }
 153
 154 void ValidateScanScript(j_compress_ptr cinfo) {
 155   // Mask of coefficient bits defined by the scan script, for each component
 156   // and coefficient index.
 157   uint16_t comp_mask[kMaxComponents][DCTSIZE2] = {};
 158   static constexpr int kMaxRefinementBit = 10;
 159
 160   for (int i = 0; i < cinfo->num_scans; ++i) {
 161     const jpeg_scan_info& si = cinfo->scan_info[i];
 162     if (si.comps_in_scan < 1 || si.comps_in_scan > MAX_COMPS_IN_SCAN) {
 163       JPEGLI_ERROR("Invalid number of components in scan %d", si.comps_in_scan);
 164     }
 165     int last_ci = -1;
 166     for (int j = 0; j < si.comps_in_scan; ++j) {
 167       int ci = si.component_index[j];
 168       if (ci < 0 || ci >= cinfo->num_components) {
 169         JPEGLI_ERROR("Invalid component index %d in scan", ci);
 170       } else if (ci == last_ci) {
 171         JPEGLI_ERROR("Duplicate component index %d in scan", ci);
 172       } else if (ci < last_ci) {
 173         JPEGLI_ERROR("Out of order component index %d in scan", ci);
 174       }
 175       last_ci = ci;
 176     }
 177     if (si.Ss < 0 || si.Se < si.Ss || si.Se >= DCTSIZE2) {
 178       JPEGLI_ERROR("Invalid spectral range %d .. %d in scan", si.Ss, si.Se);
 179     }
 180     if (si.Ah < 0 || si.Al < 0 || si.Al > kMaxRefinementBit) {
 181       JPEGLI_ERROR("Invalid refinement bits %d/%d", si.Ah, si.Al);
 182     }
 183     if (!cinfo->progressive_mode) {
 184       if (si.Ss != 0 || si.Se != DCTSIZE2 - 1 || si.Ah != 0 || si.Al != 0) {
 185         JPEGLI_ERROR("Invalid scan for sequential mode");
 186       }
 187     } else {
 188       if (si.Ss == 0 && si.Se != 0) {
 189         JPEGLI_ERROR("DC and AC together in progressive scan");
 190       }
 191     }
 192     if (si.Ss != 0 && si.comps_in_scan != 1) {
 193       JPEGLI_ERROR("Interleaved AC only scan.");
 194     }
 195     for (int j = 0; j < si.comps_in_scan; ++j) {
 196       int ci = si.component_index[j];
 197       if (si.Ss != 0 && comp_mask[ci][0] == 0) {
 198         JPEGLI_ERROR("AC before DC in component %d of scan", ci);
 199       }
 200       for (int k = si.Ss; k <= si.Se; ++k) {
 201         if (comp_mask[ci][k] == 0) {
 202           if (si.Ah != 0) {
 203             JPEGLI_ERROR("Invalid first scan refinement bit");
 204           }
 205           comp_mask[ci][k] = ((0xffff << si.Al) & 0xffff);
 206         } else {
 207           if (comp_mask[ci][k] != ((0xffff << si.Ah) & 0xffff) ||
 208               si.Al != si.Ah - 1) {
 209             JPEGLI_ERROR("Invalid refinement bit progression.");
 210           }
 211           comp_mask[ci][k] |= 1 << si.Al;
 212         }
 213       }
 214     }
 215     if (si.comps_in_scan > 1) {
 216       size_t mcu_size = 0;
 217       for (int j = 0; j < si.comps_in_scan; ++j) {
 218         int ci = si.component_index[j];
 219         jpeg_component_info* comp = &cinfo->comp_info[ci];
 220         mcu_size += comp->h_samp_factor * comp->v_samp_factor;
 221       }
 222       if (mcu_size > C_MAX_BLOCKS_IN_MCU) {
 223         JPEGLI_ERROR("MCU size too big");
 224       }
 225     }
 226   }
 227   for (int c = 0; c < cinfo->num_components; ++c) {
 228     for (int k = 0; k < DCTSIZE2; ++k) {
 229       if (comp_mask[c][k] != 0xffff) {
 230         JPEGLI_ERROR("Incomplete scan of component %d and frequency %d", c, k);
 231       }
 232     }
 233   }
 234 }
 235
 236 void ProcessCompressionParams(j_compress_ptr cinfo) {
 237   if (cinfo->dest == nullptr) {
 238     JPEGLI_ERROR("Missing destination.");
 239   }
 240   if (cinfo->image_width < 1 || cinfo->image_height < 1 ||
 241       cinfo->input_components < 1) {
 242     JPEGLI_ERROR("Empty input image.");
 243   }
 244   if (cinfo->image_width > static_cast<int>(JPEG_MAX_DIMENSION) ||
 245       cinfo->image_height > static_cast<int>(JPEG_MAX_DIMENSION) ||
 246       cinfo->input_components > static_cast<int>(kMaxComponents)) {
 247     JPEGLI_ERROR("Input image too big.");
 248   }
 249   if (cinfo->num_components < 1 ||
 250       cinfo->num_components > static_cast<int>(kMaxComponents)) {
 251     JPEGLI_ERROR("Invalid number of components.");
 252   }
 253   if (cinfo->data_precision != kJpegPrecision) {
 254     JPEGLI_ERROR("Invalid data precision");
 255   }
 256   if (cinfo->arith_code) {
 257     JPEGLI_ERROR("Arithmetic coding is not implemented.");
 258   }
 259   if (cinfo->CCIR601_sampling) {
 260     JPEGLI_ERROR("CCIR601 sampling is not implemented.");
 261   }
 262   if (cinfo->restart_interval > 65535u) {
 263     JPEGLI_ERROR("Restart interval too big");
 264   }
 265   if (cinfo->smoothing_factor < 0 || cinfo->smoothing_factor > 100) {
 266     JPEGLI_ERROR("Invalid smoothing factor %d", cinfo->smoothing_factor);
 267   }
 268   jpeg_comp_master* m = cinfo->master;
 269   cinfo->max_h_samp_factor = cinfo->max_v_samp_factor = 1;
 270   for (int c = 0; c < cinfo->num_components; ++c) {
 271     jpeg_component_info* comp = &cinfo->comp_info[c];
 272     if (comp->component_index != c) {
 273       JPEGLI_ERROR("Invalid component index");
 274     }
 275     for (int j = 0; j < c; ++j) {
 276       if (cinfo->comp_info[j].component_id == comp->component_id) {
 277         JPEGLI_ERROR("Duplicate component id %d", comp->component_id);
 278       }
 279     }
 280     if (comp->h_samp_factor <= 0 || comp->v_samp_factor <= 0 ||
 281         comp->h_samp_factor > MAX_SAMP_FACTOR ||
 282         comp->v_samp_factor > MAX_SAMP_FACTOR) {
 283       JPEGLI_ERROR("Invalid sampling factor %d x %d", comp->h_samp_factor,
 284                    comp->v_samp_factor);
 285     }
 286     if (cinfo->num_components == 1) {
 287       // Force samp factors to 1x1 for single-component images.
 288       comp->h_samp_factor = comp->v_samp_factor = 1;
 289     }
 290     cinfo->max_h_samp_factor =
 291         std::max(comp->h_samp_factor, cinfo->max_h_samp_factor);
 292     cinfo->max_v_samp_factor =
 293         std::max(comp->v_samp_factor, cinfo->max_v_samp_factor);
 294   }
 295   size_t iMCU_width = DCTSIZE * cinfo->max_h_samp_factor;
 296   size_t iMCU_height = DCTSIZE * cinfo->max_v_samp_factor;
 297   size_t total_iMCU_cols = DivCeil(cinfo->image_width, iMCU_width);
 298   cinfo->total_iMCU_rows = DivCeil(cinfo->image_height, iMCU_height);
 299   m->xsize_blocks = total_iMCU_cols * cinfo->max_h_samp_factor;
 300   m->ysize_blocks = cinfo->total_iMCU_rows * cinfo->max_v_samp_factor;
 301
 302   size_t blocks_per_iMCU = 0;
 303   for (int c = 0; c < cinfo->num_components; ++c) {
 304     jpeg_component_info* comp = &cinfo->comp_info[c];
 305     if (cinfo->max_h_samp_factor % comp->h_samp_factor != 0 ||
 306         cinfo->max_v_samp_factor % comp->v_samp_factor != 0) {
 307       JPEGLI_ERROR("Non-integral sampling ratios are not supported.");
 308     }
 309     m->h_factor[c] = cinfo->max_h_samp_factor / comp->h_samp_factor;
 310     m->v_factor[c] = cinfo->max_v_samp_factor / comp->v_samp_factor;
 311     comp->downsampled_width = DivCeil(cinfo->image_width, m->h_factor[c]);
 312     comp->downsampled_height = DivCeil(cinfo->image_height, m->v_factor[c]);
 313     comp->width_in_blocks = DivCeil(comp->downsampled_width, DCTSIZE);
 314     comp->height_in_blocks = DivCeil(comp->downsampled_height, DCTSIZE);
 315     blocks_per_iMCU += comp->h_samp_factor * comp->v_samp_factor;
 316   }
 317   m->blocks_per_iMCU_row = total_iMCU_cols * blocks_per_iMCU;
 318   // Disable adaptive quantization for subsampled luma channel.
 319   int y_channel = cinfo->jpeg_color_space == JCS_RGB ? 1 : 0;
 320   jpeg_component_info* y_comp = &cinfo->comp_info[y_channel];
 321   if (y_comp->h_samp_factor != cinfo->max_h_samp_factor ||
 322       y_comp->v_samp_factor != cinfo->max_v_samp_factor) {
 323     m->use_adaptive_quantization = false;
 324   }
 325   if (cinfo->scan_info == nullptr) {
 326     SetDefaultScanScript(cinfo);
 327   }
 328   cinfo->progressive_mode = TO_JXL_BOOL(cinfo->scan_info->Ss != 0 ||
 329                                         cinfo->scan_info->Se != DCTSIZE2 - 1);
 330   ValidateScanScript(cinfo);
 331   m->scan_token_info =
 332       Allocate<ScanTokenInfo>(cinfo, cinfo->num_scans, JPOOL_IMAGE);
 333   memset(m->scan_token_info, 0, cinfo->num_scans * sizeof(ScanTokenInfo));
 334   m->ac_ctx_offset = Allocate<uint8_t>(cinfo, cinfo->num_scans, JPOOL_IMAGE);
 335   size_t num_ac_contexts = 0;
 336   for (int i = 0; i < cinfo->num_scans; ++i) {
 337     const jpeg_scan_info* scan_info = &cinfo->scan_info[i];
 338     m->ac_ctx_offset[i] = 4 + num_ac_contexts;
 339     if (scan_info->Se > 0) {
 340       num_ac_contexts += scan_info->comps_in_scan;
 341     }
 342     if (num_ac_contexts > 252) {
 343       JPEGLI_ERROR("Too many AC scans in image");
 344     }
 345     ScanTokenInfo* sti = &m->scan_token_info[i];
 346     if (scan_info->comps_in_scan == 1) {
 347       int comp_idx = scan_info->component_index[0];
 348       jpeg_component_info* comp = &cinfo->comp_info[comp_idx];
 349       sti->MCUs_per_row = comp->width_in_blocks;
 350       sti->MCU_rows_in_scan = comp->height_in_blocks;
 351       sti->blocks_in_MCU = 1;
 352     } else {
 353       sti->MCUs_per_row =
 354           DivCeil(cinfo->image_width, DCTSIZE * cinfo->max_h_samp_factor);
 355       sti->MCU_rows_in_scan =
 356           DivCeil(cinfo->image_height, DCTSIZE * cinfo->max_v_samp_factor);
 357       sti->blocks_in_MCU = 0;
 358       for (int j = 0; j < scan_info->comps_in_scan; ++j) {
 359         int comp_idx = scan_info->component_index[j];
 360         jpeg_component_info* comp = &cinfo->comp_info[comp_idx];
 361         sti->blocks_in_MCU += comp->h_samp_factor * comp->v_samp_factor;
 362       }
 363     }
 364     size_t num_MCUs = sti->MCU_rows_in_scan * sti->MCUs_per_row;
 365     sti->num_blocks = num_MCUs * sti->blocks_in_MCU;
 366     if (cinfo->restart_in_rows <= 0) {
 367       sti->restart_interval = cinfo->restart_interval;
 368     } else {
 369       sti->restart_interval =
 370           std::min<size_t>(sti->MCUs_per_row * cinfo->restart_in_rows, 65535u);
 371     }
 372     sti->num_restarts = sti->restart_interval > 0
 373                             ? DivCeil(num_MCUs, sti->restart_interval)
 374                             : 1;
 375     sti->restarts = Allocate<size_t>(cinfo, sti->num_restarts, JPOOL_IMAGE);
 376   }
 377   m->num_contexts = 4 + num_ac_contexts;
 378 }
 379
 380 bool IsStreamingSupported(j_compress_ptr cinfo) {
 381   if (cinfo->global_state == kEncWriteCoeffs) {
 382     return false;
 383   }
 384   // TODO(szabadka) Remove this restriction.
 385   if (cinfo->restart_interval > 0 || cinfo->restart_in_rows > 0) {
 386     return false;
 387   }
 388   if (cinfo->num_scans > 1) {
 389     return false;
 390   }
 391   if (cinfo->master->psnr_target > 0) {
 392     return false;
 393   }
 394   return true;
 395 }
 396
 397 void AllocateBuffers(j_compress_ptr cinfo) {
 398   jpeg_comp_master* m = cinfo->master;
 399   memset(m->last_dc_coeff, 0, sizeof(m->last_dc_coeff));
 400   if (!IsStreamingSupported(cinfo) || cinfo->optimize_coding) {
 401     int ysize_blocks = DivCeil(cinfo->image_height, DCTSIZE);
 402     int num_arrays = cinfo->num_scans * ysize_blocks;
 403     m->token_arrays = Allocate<TokenArray>(cinfo, num_arrays, JPOOL_IMAGE);
 404     m->cur_token_array = 0;
 405     memset(m->token_arrays, 0, num_arrays * sizeof(TokenArray));
 406     m->num_tokens = 0;
 407     m->total_num_tokens = 0;
 408   }
 409   if (cinfo->global_state == kEncWriteCoeffs) {
 410     return;
 411   }
 412   size_t iMCU_width = DCTSIZE * cinfo->max_h_samp_factor;
 413   size_t iMCU_height = DCTSIZE * cinfo->max_v_samp_factor;
 414   size_t total_iMCU_cols = DivCeil(cinfo->image_width, iMCU_width);
 415   size_t xsize_full = total_iMCU_cols * iMCU_width;
 416   size_t ysize_full = 3 * iMCU_height;
 417   if (!cinfo->raw_data_in) {
 418     int num_all_components =
 419         std::max(cinfo->input_components, cinfo->num_components);
 420     for (int c = 0; c < num_all_components; ++c) {
 421       m->input_buffer[c].Allocate(cinfo, ysize_full, xsize_full);
 422     }
 423   }
 424   for (int c = 0; c < cinfo->num_components; ++c) {
 425     jpeg_component_info* comp = &cinfo->comp_info[c];
 426     size_t xsize = total_iMCU_cols * comp->h_samp_factor * DCTSIZE;
 427     size_t ysize = 3 * comp->v_samp_factor * DCTSIZE;
 428     if (cinfo->raw_data_in) {
 429       m->input_buffer[c].Allocate(cinfo, ysize, xsize);
 430     }
 431     m->smooth_input[c] = &m->input_buffer[c];
 432     if (!cinfo->raw_data_in && cinfo->smoothing_factor) {
 433       m->smooth_input[c] = Allocate<RowBuffer<float>>(cinfo, 1, JPOOL_IMAGE);
 434       m->smooth_input[c]->Allocate(cinfo, ysize_full, xsize_full);
 435     }
 436     m->raw_data[c] = m->smooth_input[c];
 437     if (!cinfo->raw_data_in && (m->h_factor[c] > 1 || m->v_factor[c] > 1)) {
 438       m->raw_data[c] = Allocate<RowBuffer<float>>(cinfo, 1, JPOOL_IMAGE);
 439       m->raw_data[c]->Allocate(cinfo, ysize, xsize);
 440     }
 441     m->quant_mul[c] = Allocate<float>(cinfo, DCTSIZE2, JPOOL_IMAGE_ALIGNED);
 442   }
 443   m->dct_buffer = Allocate<float>(cinfo, 2 * DCTSIZE2, JPOOL_IMAGE_ALIGNED);
 444   m->block_tmp = Allocate<int32_t>(cinfo, DCTSIZE2 * 4, JPOOL_IMAGE_ALIGNED);
 445   if (!IsStreamingSupported(cinfo)) {
 446     m->coeff_buffers =
 447         Allocate<jvirt_barray_ptr>(cinfo, cinfo->num_components, JPOOL_IMAGE);
 448     for (int c = 0; c < cinfo->num_components; ++c) {
 449       jpeg_component_info* comp = &cinfo->comp_info[c];
 450       const size_t xsize_blocks = comp->width_in_blocks;
 451       const size_t ysize_blocks = comp->height_in_blocks;
 452       m->coeff_buffers[c] = (*cinfo->mem->request_virt_barray)(
 453           reinterpret_cast<j_common_ptr>(cinfo), JPOOL_IMAGE,
 454           /*pre_zero=*/FALSE, xsize_blocks, ysize_blocks, comp->v_samp_factor);
 455     }
 456   }
 457   if (m->use_adaptive_quantization) {
 458     int y_channel = cinfo->jpeg_color_space == JCS_RGB ? 1 : 0;
 459     jpeg_component_info* y_comp = &cinfo->comp_info[y_channel];
 460     const size_t xsize_blocks = y_comp->width_in_blocks;
 461     const size_t vecsize = VectorSize();
 462     const size_t xsize_padded = DivCeil(2 * xsize_blocks, vecsize) * vecsize;
 463     m->diff_buffer =
 464         Allocate<float>(cinfo, xsize_blocks * DCTSIZE + 8, JPOOL_IMAGE_ALIGNED);
 465     m->fuzzy_erosion_tmp.Allocate(cinfo, 2, xsize_padded);
 466     m->pre_erosion.Allocate(cinfo, 6 * cinfo->max_v_samp_factor, xsize_padded);
 467     size_t qf_height = cinfo->max_v_samp_factor;
 468     if (m->psnr_target > 0) {
 469       qf_height *= cinfo->total_iMCU_rows;
 470     }
 471     m->quant_field.Allocate(cinfo, qf_height, xsize_blocks);
 472   } else {
 473     m->quant_field.Allocate(cinfo, 1, m->xsize_blocks);
 474     m->quant_field.FillRow(0, 0, m->xsize_blocks);
 475   }
 476   for (int c = 0; c < cinfo->num_components; ++c) {
 477     m->zero_bias_offset[c] =
 478         Allocate<float>(cinfo, DCTSIZE2, JPOOL_IMAGE_ALIGNED);
 479     m->zero_bias_mul[c] = Allocate<float>(cinfo, DCTSIZE2, JPOOL_IMAGE_ALIGNED);
 480     memset(m->zero_bias_mul[c], 0, DCTSIZE2 * sizeof(float));
 481     memset(m->zero_bias_offset[c], 0, DCTSIZE2 * sizeof(float));
 482   }
 483 }
 484
 485 void InitProgressMonitor(j_compress_ptr cinfo) {
 486   if (cinfo->progress == nullptr) {
 487     return;
 488   }
 489   if (IsStreamingSupported(cinfo)) {
 490     // We have only one input pass.
 491     cinfo->progress->total_passes = 1;
 492   } else {
 493     // We have one input pass, a histogram pass for each scan, and an encode
 494     // pass for each scan.
 495     cinfo->progress->total_passes = 1 + 2 * cinfo->num_scans;
 496   }
 497 }
 498
 499 // Common setup code between streaming and transcoding code paths. Called in
 500 // both jpegli_start_compress() and jpegli_write_coefficients().
 501 void InitCompress(j_compress_ptr cinfo, boolean write_all_tables) {
 502   jpeg_comp_master* m = cinfo->master;
 503   (*cinfo->err->reset_error_mgr)(reinterpret_cast<j_common_ptr>(cinfo));
 504   ProcessCompressionParams(cinfo);
 505   InitProgressMonitor(cinfo);
 506   AllocateBuffers(cinfo);
 507   if (cinfo->global_state != kEncWriteCoeffs) {
 508     ChooseInputMethod(cinfo);
 509     if (!cinfo->raw_data_in) {
 510       ChooseColorTransform(cinfo);
 511       ChooseDownsampleMethods(cinfo);
 512     }
 513     QuantPass pass = m->psnr_target > 0 ? QuantPass::SEARCH_FIRST_PASS
 514                                         : QuantPass::NO_SEARCH;
 515     InitQuantizer(cinfo, pass);
 516   }
 517   if (write_all_tables) {
 518     jpegli_suppress_tables(cinfo, FALSE);
 519   }
 520   if (!cinfo->optimize_coding && !cinfo->progressive_mode) {
 521     CopyHuffmanTables(cinfo);
 522     InitEntropyCoder(cinfo);
 523   }
 524   (*cinfo->dest->init_destination)(cinfo);
 525   WriteFileHeader(cinfo);
 526   JpegBitWriterInit(cinfo);
 527   m->next_iMCU_row = 0;
 528   m->last_restart_interval = 0;
 529   m->next_dht_index = 0;
 530 }
 531
 532 //
 533 // Input streaming
 534 //
 535
 536 void ProgressMonitorInputPass(j_compress_ptr cinfo) {
 537   if (cinfo->progress == nullptr) {
 538     return;
 539   }
 540   cinfo->progress->completed_passes = 0;
 541   cinfo->progress->pass_counter = cinfo->next_scanline;
 542   cinfo->progress->pass_limit = cinfo->image_height;
 543   (*cinfo->progress->progress_monitor)(reinterpret_cast<j_common_ptr>(cinfo));
 544 }
 545
 546 void ReadInputRow(j_compress_ptr cinfo, const uint8_t* scanline,
 547                   float* row[kMaxComponents]) {
 548   jpeg_comp_master* m = cinfo->master;
 549   int num_all_components =
 550       std::max(cinfo->input_components, cinfo->num_components);
 551   for (int c = 0; c < num_all_components; ++c) {
 552     row[c] = m->input_buffer[c].Row(m->next_input_row);
 553   }
 554   ++m->next_input_row;
 555   if (scanline == nullptr) {
 556     for (int c = 0; c < cinfo->input_components; ++c) {
 557       memset(row[c], 0, cinfo->image_width * sizeof(row[c][0]));
 558     }
 559     return;
 560   }
 561   (*m->input_method)(scanline, cinfo->image_width, row);
 562 }
 563
 564 void PadInputBuffer(j_compress_ptr cinfo, float* row[kMaxComponents]) {
 565   jpeg_comp_master* m = cinfo->master;
 566   const size_t len0 = cinfo->image_width;
 567   const size_t len1 = m->xsize_blocks * DCTSIZE;
 568   for (int c = 0; c < cinfo->num_components; ++c) {
 569     // Pad row to a multiple of the iMCU width, plus create a border of 1
 570     // repeated pixel for adaptive quant field calculation.
 571     float last_val = row[c][len0 - 1];
 572     for (size_t x = len0; x <= len1; ++x) {
 573       row[c][x] = last_val;
 574     }
 575     row[c][-1] = row[c][0];
 576   }
 577   if (m->next_input_row == cinfo->image_height) {
 578     size_t num_rows = m->ysize_blocks * DCTSIZE - cinfo->image_height;
 579     for (size_t i = 0; i < num_rows; ++i) {
 580       for (int c = 0; c < cinfo->num_components; ++c) {
 581         float* dest = m->input_buffer[c].Row(m->next_input_row) - 1;
 582         memcpy(dest, row[c] - 1, (len1 + 2) * sizeof(dest[0]));
 583       }
 584       ++m->next_input_row;
 585     }
 586   }
 587 }
 588
 589 void ProcessiMCURow(j_compress_ptr cinfo) {
 590   JPEGLI_CHECK(cinfo->master->next_iMCU_row < cinfo->total_iMCU_rows);
 591   if (!cinfo->raw_data_in) {
 592     ApplyInputSmoothing(cinfo);
 593     DownsampleInputBuffer(cinfo);
 594   }
 595   ComputeAdaptiveQuantField(cinfo);
 596   if (IsStreamingSupported(cinfo)) {
 597     if (cinfo->optimize_coding) {
 598       ComputeTokensForiMCURow(cinfo);
 599     } else {
 600       WriteiMCURow(cinfo);
 601     }
 602   } else {
 603     ComputeCoefficientsForiMCURow(cinfo);
 604   }
 605   ++cinfo->master->next_iMCU_row;
 606 }
 607
 608 void ProcessiMCURows(j_compress_ptr cinfo) {
 609   jpeg_comp_master* m = cinfo->master;
 610   size_t iMCU_height = DCTSIZE * cinfo->max_v_samp_factor;
 611   // To have context rows both above and below the current iMCU row, we delay
 612   // processing the first iMCU row and process two iMCU rows after we receive
 613   // the last input row.
 614   if (m->next_input_row % iMCU_height == 0 && m->next_input_row > iMCU_height) {
 615     ProcessiMCURow(cinfo);
 616   }
 617   if (m->next_input_row >= cinfo->image_height) {
 618     ProcessiMCURow(cinfo);
 619   }
 620 }
 621
 622 //
 623 // Non-streaming part
 624 //
 625
 626 void ZigZagShuffleBlocks(j_compress_ptr cinfo) {
 627   JCOEF tmp[DCTSIZE2];
 628   for (int c = 0; c < cinfo->num_components; ++c) {
 629     jpeg_component_info* comp = &cinfo->comp_info[c];
 630     for (JDIMENSION by = 0; by < comp->height_in_blocks; ++by) {
 631       JBLOCKARRAY blocks = GetBlockRow(cinfo, c, by);
 632       for (JDIMENSION bx = 0; bx < comp->width_in_blocks; ++bx) {
 633         JCOEF* block = &blocks[0][bx][0];
 634         for (int k = 0; k < DCTSIZE2; ++k) {
 635           tmp[k] = block[kJPEGNaturalOrder[k]];
 636         }
 637         memcpy(block, tmp, sizeof(tmp));
 638       }
 639     }
 640   }
 641 }
 642
 643 }  // namespace jpegli
 644
 645 //
 646 // Parameter setup
 647 //
 648
 649 void jpegli_CreateCompress(j_compress_ptr cinfo, int version,
 650                            size_t structsize) {
 651   cinfo->mem = nullptr;
 652   if (structsize != sizeof(*cinfo)) {
 653     JPEGLI_ERROR("jpegli_compress_struct has wrong size.");
 654   }
 655   jpegli::InitMemoryManager(reinterpret_cast<j_common_ptr>(cinfo));
 656   cinfo->progress = nullptr;
 657   cinfo->is_decompressor = FALSE;
 658   cinfo->global_state = jpegli::kEncStart;
 659   cinfo->dest = nullptr;
 660   cinfo->image_width = 0;
 661   cinfo->image_height = 0;
 662   cinfo->input_components = 0;
 663   cinfo->in_color_space = JCS_UNKNOWN;
 664   cinfo->input_gamma = 1.0f;
 665   cinfo->num_components = 0;
 666   cinfo->jpeg_color_space = JCS_UNKNOWN;
 667   cinfo->comp_info = nullptr;
 668   for (auto& quant_tbl_ptr : cinfo->quant_tbl_ptrs) {
 669     quant_tbl_ptr = nullptr;
 670   }
 671   for (int i = 0; i < NUM_HUFF_TBLS; ++i) {
 672     cinfo->dc_huff_tbl_ptrs[i] = nullptr;
 673     cinfo->ac_huff_tbl_ptrs[i] = nullptr;
 674   }
 675   memset(cinfo->arith_dc_L, 0, sizeof(cinfo->arith_dc_L));
 676   memset(cinfo->arith_dc_U, 0, sizeof(cinfo->arith_dc_U));
 677   memset(cinfo->arith_ac_K, 0, sizeof(cinfo->arith_ac_K));
 678   cinfo->write_Adobe_marker = FALSE;
 679   cinfo->master = jpegli::Allocate<jpeg_comp_master>(cinfo, 1);
 680   jpegli::InitializeCompressParams(cinfo);
 681   cinfo->master->force_baseline = true;
 682   cinfo->master->xyb_mode = false;
 683   cinfo->master->cicp_transfer_function = 2;  // unknown transfer function code
 684   cinfo->master->use_std_tables = false;
 685   cinfo->master->use_adaptive_quantization = true;
 686   cinfo->master->progressive_level = jpegli::kDefaultProgressiveLevel;
 687   cinfo->master->data_type = JPEGLI_TYPE_UINT8;
 688   cinfo->master->endianness = JPEGLI_NATIVE_ENDIAN;
 689   cinfo->master->coeff_buffers = nullptr;
 690 }
 691
 692 void jpegli_set_xyb_mode(j_compress_ptr cinfo) {
 693   CheckState(cinfo, jpegli::kEncStart);
 694   cinfo->master->xyb_mode = true;
 695 }
 696
 697 void jpegli_set_cicp_transfer_function(j_compress_ptr cinfo, int code) {
 698   CheckState(cinfo, jpegli::kEncStart);
 699   cinfo->master->cicp_transfer_function = code;
 700 }
 701
 702 void jpegli_set_defaults(j_compress_ptr cinfo) {
 703   CheckState(cinfo, jpegli::kEncStart);
 704   jpegli::InitializeCompressParams(cinfo);
 705   jpegli_default_colorspace(cinfo);
 706   jpegli_set_quality(cinfo, 90, TRUE);
 707   jpegli_set_progressive_level(cinfo, jpegli::kDefaultProgressiveLevel);
 708   jpegli::AddStandardHuffmanTables(reinterpret_cast<j_common_ptr>(cinfo),
 709                                    /*is_dc=*/false);
 710   jpegli::AddStandardHuffmanTables(reinterpret_cast<j_common_ptr>(cinfo),
 711                                    /*is_dc=*/true);
 712 }
 713
 714 void jpegli_default_colorspace(j_compress_ptr cinfo) {
 715   CheckState(cinfo, jpegli::kEncStart);
 716   if (cinfo->in_color_space == JCS_RGB && cinfo->master->xyb_mode) {
 717     jpegli_set_colorspace(cinfo, JCS_RGB);
 718     return;
 719   }
 720   switch (cinfo->in_color_space) {
 721     case JCS_GRAYSCALE:
 722       jpegli_set_colorspace(cinfo, JCS_GRAYSCALE);
 723       break;
 724     case JCS_RGB:
 725 #ifdef JCS_EXTENSIONS
 726     case JCS_EXT_RGB:
 727     case JCS_EXT_BGR:
 728     case JCS_EXT_RGBX:
 729     case JCS_EXT_BGRX:
 730     case JCS_EXT_XRGB:
 731     case JCS_EXT_XBGR:
 732 #endif
 733 #if JCS_ALPHA_EXTENSIONS
 734     case JCS_EXT_RGBA:
 735     case JCS_EXT_BGRA:
 736     case JCS_EXT_ARGB:
 737     case JCS_EXT_ABGR:
 738 #endif
 739       jpegli_set_colorspace(cinfo, JCS_YCbCr);
 740       break;
 741     case JCS_YCbCr:
 742       jpegli_set_colorspace(cinfo, JCS_YCbCr);
 743       break;
 744     case JCS_CMYK:
 745       jpegli_set_colorspace(cinfo, JCS_CMYK);
 746       break;
 747     case JCS_YCCK:
 748       jpegli_set_colorspace(cinfo, JCS_YCCK);
 749       break;
 750     case JCS_UNKNOWN:
 751       jpegli_set_colorspace(cinfo, JCS_UNKNOWN);
 752       break;
 753     default:
 754       JPEGLI_ERROR("Unsupported input colorspace %d", cinfo->in_color_space);
 755   }
 756 }
 757
 758 void jpegli_set_colorspace(j_compress_ptr cinfo, J_COLOR_SPACE colorspace) {
 759   CheckState(cinfo, jpegli::kEncStart);
 760   cinfo->jpeg_color_space = colorspace;
 761   switch (colorspace) {
 762     case JCS_GRAYSCALE:
 763       cinfo->num_components = 1;
 764       break;
 765     case JCS_RGB:
 766     case JCS_YCbCr:
 767       cinfo->num_components = 3;
 768       break;
 769     case JCS_CMYK:
 770     case JCS_YCCK:
 771       cinfo->num_components = 4;
 772       break;
 773     case JCS_UNKNOWN:
 774       cinfo->num_components =
 775           std::min<int>(jpegli::kMaxComponents, cinfo->input_components);
 776       break;
 777     default:
 778       JPEGLI_ERROR("Unsupported jpeg colorspace %d", colorspace);
 779   }
 780   // Adobe marker is only needed to distinguish CMYK and YCCK JPEGs.
 781   cinfo->write_Adobe_marker = TO_JXL_BOOL(cinfo->jpeg_color_space == JCS_YCCK);
 782   if (cinfo->comp_info == nullptr) {
 783     cinfo->comp_info =
 784         jpegli::Allocate<jpeg_component_info>(cinfo, MAX_COMPONENTS);
 785   }
 786   memset(cinfo->comp_info, 0,
 787          jpegli::kMaxComponents * sizeof(jpeg_component_info));
 788   for (int c = 0; c < cinfo->num_components; ++c) {
 789     jpeg_component_info* comp = &cinfo->comp_info[c];
 790     comp->component_index = c;
 791     comp->component_id = c + 1;
 792     comp->h_samp_factor = 1;
 793     comp->v_samp_factor = 1;
 794     comp->quant_tbl_no = 0;
 795     comp->dc_tbl_no = 0;
 796     comp->ac_tbl_no = 0;
 797   }
 798   if (colorspace == JCS_RGB) {
 799     cinfo->comp_info[0].component_id = 'R';
 800     cinfo->comp_info[1].component_id = 'G';
 801     cinfo->comp_info[2].component_id = 'B';
 802     if (cinfo->master->xyb_mode) {
 803       // Subsample blue channel.
 804       cinfo->comp_info[0].h_samp_factor = cinfo->comp_info[0].v_samp_factor = 2;
 805       cinfo->comp_info[1].h_samp_factor = cinfo->comp_info[1].v_samp_factor = 2;
 806       cinfo->comp_info[2].h_samp_factor = cinfo->comp_info[2].v_samp_factor = 1;
 807       // Use separate quantization tables for each component
 808       cinfo->comp_info[1].quant_tbl_no = 1;
 809       cinfo->comp_info[2].quant_tbl_no = 2;
 810     }
 811   } else if (colorspace == JCS_CMYK) {
 812     cinfo->comp_info[0].component_id = 'C';
 813     cinfo->comp_info[1].component_id = 'M';
 814     cinfo->comp_info[2].component_id = 'Y';
 815     cinfo->comp_info[3].component_id = 'K';
 816   } else if (colorspace == JCS_YCbCr || colorspace == JCS_YCCK) {
 817     // Use separate quantization and Huffman tables for luma and chroma
 818     cinfo->comp_info[1].quant_tbl_no = 1;
 819     cinfo->comp_info[2].quant_tbl_no = 1;
 820     cinfo->comp_info[1].dc_tbl_no = cinfo->comp_info[1].ac_tbl_no = 1;
 821     cinfo->comp_info[2].dc_tbl_no = cinfo->comp_info[2].ac_tbl_no = 1;
 822     // Use chroma subsampling by default
 823     cinfo->comp_info[0].h_samp_factor = cinfo->comp_info[0].v_samp_factor = 2;
 824     if (colorspace == JCS_YCCK) {
 825       cinfo->comp_info[3].h_samp_factor = cinfo->comp_info[3].v_samp_factor = 2;
 826     }
 827   }
 828 }
 829
 830 void jpegli_set_distance(j_compress_ptr cinfo, float distance,
 831                          boolean force_baseline) {
 832   CheckState(cinfo, jpegli::kEncStart);
 833   cinfo->master->force_baseline = FROM_JXL_BOOL(force_baseline);
 834   float distances[NUM_QUANT_TBLS] = {distance, distance, distance};
 835   jpegli::SetQuantMatrices(cinfo, distances, /*add_two_chroma_tables=*/true);
 836 }
 837
 838 float jpegli_quality_to_distance(int quality) {
 839   return (quality >= 100  ? 0.01f
 840           : quality >= 30 ? 0.1f + (100 - quality) * 0.09f
 841                           : 53.0f / 3000.0f * quality * quality -
 842                                 23.0f / 20.0f * quality + 25.0f);
 843 }
 844
 845 void jpegli_set_psnr(j_compress_ptr cinfo, float psnr, float tolerance,
 846                      float min_distance, float max_distance) {
 847   CheckState(cinfo, jpegli::kEncStart);
 848   cinfo->master->psnr_target = psnr;
 849   cinfo->master->psnr_tolerance = tolerance;
 850   cinfo->master->min_distance = min_distance;
 851   cinfo->master->max_distance = max_distance;
 852 }
 853
 854 void jpegli_set_quality(j_compress_ptr cinfo, int quality,
 855                         boolean force_baseline) {
 856   CheckState(cinfo, jpegli::kEncStart);
 857   cinfo->master->force_baseline = FROM_JXL_BOOL(force_baseline);
 858   float distance = jpegli_quality_to_distance(quality);
 859   float distances[NUM_QUANT_TBLS] = {distance, distance, distance};
 860   jpegli::SetQuantMatrices(cinfo, distances, /*add_two_chroma_tables=*/false);
 861 }
 862
 863 void jpegli_set_linear_quality(j_compress_ptr cinfo, int scale_factor,
 864                                boolean force_baseline) {
 865   CheckState(cinfo, jpegli::kEncStart);
 866   cinfo->master->force_baseline = FROM_JXL_BOOL(force_baseline);
 867   float distance = jpegli::LinearQualityToDistance(scale_factor);
 868   float distances[NUM_QUANT_TBLS] = {distance, distance, distance};
 869   jpegli::SetQuantMatrices(cinfo, distances, /*add_two_chroma_tables=*/false);
 870 }
 871
 872 #if JPEG_LIB_VERSION >= 70
 873 void jpegli_default_qtables(j_compress_ptr cinfo, boolean force_baseline) {
 874   CheckState(cinfo, jpegli::kEncStart);
 875   cinfo->master->force_baseline = force_baseline;
 876   float distances[NUM_QUANT_TBLS];
 877   for (int i = 0; i < NUM_QUANT_TBLS; ++i) {
 878     distances[i] = jpegli::LinearQualityToDistance(cinfo->q_scale_factor[i]);
 879   }
 880   jpegli::SetQuantMatrices(cinfo, distances, /*add_two_chroma_tables=*/false);
 881 }
 882 #endif
 883
 884 int jpegli_quality_scaling(int quality) {
 885   quality = std::min(100, std::max(1, quality));
 886   return quality < 50 ? 5000 / quality : 200 - 2 * quality;
 887 }
 888
 889 void jpegli_use_standard_quant_tables(j_compress_ptr cinfo) {
 890   CheckState(cinfo, jpegli::kEncStart);
 891   cinfo->master->use_std_tables = true;
 892 }
 893
 894 void jpegli_add_quant_table(j_compress_ptr cinfo, int which_tbl,
 895                             const unsigned int* basic_table, int scale_factor,
 896                             boolean force_baseline) {
 897   CheckState(cinfo, jpegli::kEncStart);
 898   if (which_tbl < 0 || which_tbl > NUM_QUANT_TBLS) {
 899     JPEGLI_ERROR("Invalid quant table index %d", which_tbl);
 900   }
 901   if (cinfo->quant_tbl_ptrs[which_tbl] == nullptr) {
 902     cinfo->quant_tbl_ptrs[which_tbl] =
 903         jpegli_alloc_quant_table(reinterpret_cast<j_common_ptr>(cinfo));
 904   }
 905   int max_qval = force_baseline ? 255 : 32767U;
 906   JQUANT_TBL* quant_table = cinfo->quant_tbl_ptrs[which_tbl];
 907   for (int k = 0; k < DCTSIZE2; ++k) {
 908     int qval = (basic_table[k] * scale_factor + 50) / 100;
 909     qval = std::max(1, std::min(qval, max_qval));
 910     quant_table->quantval[k] = qval;
 911   }
 912   quant_table->sent_table = FALSE;
 913 }
 914
 915 void jpegli_enable_adaptive_quantization(j_compress_ptr cinfo, boolean value) {
 916   CheckState(cinfo, jpegli::kEncStart);
 917   cinfo->master->use_adaptive_quantization = FROM_JXL_BOOL(value);
 918 }
 919
 920 void jpegli_simple_progression(j_compress_ptr cinfo) {
 921   CheckState(cinfo, jpegli::kEncStart);
 922   jpegli_set_progressive_level(cinfo, 2);
 923 }
 924
 925 void jpegli_set_progressive_level(j_compress_ptr cinfo, int level) {
 926   CheckState(cinfo, jpegli::kEncStart);
 927   if (level < 0) {
 928     JPEGLI_ERROR("Invalid progressive level %d", level);
 929   }
 930   cinfo->master->progressive_level = level;
 931 }
 932
 933 void jpegli_set_input_format(j_compress_ptr cinfo, JpegliDataType data_type,
 934                              JpegliEndianness endianness) {
 935   CheckState(cinfo, jpegli::kEncStart);
 936   switch (data_type) {
 937     case JPEGLI_TYPE_UINT8:
 938     case JPEGLI_TYPE_UINT16:
 939     case JPEGLI_TYPE_FLOAT:
 940       cinfo->master->data_type = data_type;
 941       break;
 942     default:
 943       JPEGLI_ERROR("Unsupported data type %d", data_type);
 944   }
 945   switch (endianness) {
 946     case JPEGLI_NATIVE_ENDIAN:
 947     case JPEGLI_LITTLE_ENDIAN:
 948     case JPEGLI_BIG_ENDIAN:
 949       cinfo->master->endianness = endianness;
 950       break;
 951     default:
 952       JPEGLI_ERROR("Unsupported endianness %d", endianness);
 953   }
 954 }
 955
 956 #if JPEG_LIB_VERSION >= 70
 957 void jpegli_calc_jpeg_dimensions(j_compress_ptr cinfo) {
 958   // Since input scaling is not supported, we just copy the image dimensions.
 959   cinfo->jpeg_width = cinfo->image_width;
 960   cinfo->jpeg_height = cinfo->image_height;
 961 }
 962 #endif
 963
 964 void jpegli_copy_critical_parameters(j_decompress_ptr srcinfo,
 965                                      j_compress_ptr dstinfo) {
 966   CheckState(dstinfo, jpegli::kEncStart);
 967   // Image parameters.
 968   dstinfo->image_width = srcinfo->image_width;
 969   dstinfo->image_height = srcinfo->image_height;
 970   dstinfo->input_components = srcinfo->num_components;
 971   dstinfo->in_color_space = srcinfo->jpeg_color_space;
 972   dstinfo->input_gamma = srcinfo->output_gamma;
 973   // Compression parameters.
 974   jpegli_set_defaults(dstinfo);
 975   jpegli_set_colorspace(dstinfo, srcinfo->jpeg_color_space);
 976   if (dstinfo->num_components != srcinfo->num_components) {
 977     const auto& cinfo = dstinfo;
 978     JPEGLI_ERROR("Mismatch between src colorspace and components");
 979   }
 980   dstinfo->data_precision = srcinfo->data_precision;
 981   dstinfo->CCIR601_sampling = srcinfo->CCIR601_sampling;
 982   dstinfo->JFIF_major_version = srcinfo->JFIF_major_version;
 983   dstinfo->JFIF_minor_version = srcinfo->JFIF_minor_version;
 984   dstinfo->density_unit = srcinfo->density_unit;
 985   dstinfo->X_density = srcinfo->X_density;
 986   dstinfo->Y_density = srcinfo->Y_density;
 987   for (int c = 0; c < dstinfo->num_components; ++c) {
 988     jpeg_component_info* srccomp = &srcinfo->comp_info[c];
 989     jpeg_component_info* dstcomp = &dstinfo->comp_info[c];
 990     dstcomp->component_id = srccomp->component_id;
 991     dstcomp->h_samp_factor = srccomp->h_samp_factor;
 992     dstcomp->v_samp_factor = srccomp->v_samp_factor;
 993     dstcomp->quant_tbl_no = srccomp->quant_tbl_no;
 994   }
 995   for (int i = 0; i < NUM_QUANT_TBLS; ++i) {
 996     if (!srcinfo->quant_tbl_ptrs[i]) continue;
 997     if (dstinfo->quant_tbl_ptrs[i] == nullptr) {
 998       dstinfo->quant_tbl_ptrs[i] = jpegli::Allocate<JQUANT_TBL>(dstinfo, 1);
 999     }
1000     memcpy(dstinfo->quant_tbl_ptrs[i], srcinfo->quant_tbl_ptrs[i],
1001            sizeof(JQUANT_TBL));
1002     dstinfo->quant_tbl_ptrs[i]->sent_table = FALSE;
1003   }
1004 }
1005
1006 void jpegli_suppress_tables(j_compress_ptr cinfo, boolean suppress) {
1007   jpegli::SetSentTableFlag(cinfo->quant_tbl_ptrs, NUM_QUANT_TBLS, suppress);
1008   jpegli::SetSentTableFlag(cinfo->dc_huff_tbl_ptrs, NUM_HUFF_TBLS, suppress);
1009   jpegli::SetSentTableFlag(cinfo->ac_huff_tbl_ptrs, NUM_HUFF_TBLS, suppress);
1010 }
1011
1012 //
1013 // Compressor initialization
1014 //
1015
1016 void jpegli_start_compress(j_compress_ptr cinfo, boolean write_all_tables) {
1017   CheckState(cinfo, jpegli::kEncStart);
1018   cinfo->global_state = jpegli::kEncHeader;
1019   jpegli::InitCompress(cinfo, write_all_tables);
1020   cinfo->next_scanline = 0;
1021   cinfo->master->next_input_row = 0;
1022 }
1023
1024 void jpegli_write_coefficients(j_compress_ptr cinfo,
1025                                jvirt_barray_ptr* coef_arrays) {
1026   CheckState(cinfo, jpegli::kEncStart);
1027   cinfo->global_state = jpegli::kEncWriteCoeffs;
1028   jpegli::InitCompress(cinfo, /*write_all_tables=*/TRUE);
1029   cinfo->master->coeff_buffers = coef_arrays;
1030   cinfo->next_scanline = cinfo->image_height;
1031   cinfo->master->next_input_row = cinfo->image_height;
1032 }
1033
1034 void jpegli_write_tables(j_compress_ptr cinfo) {
1035   CheckState(cinfo, jpegli::kEncStart);
1036   if (cinfo->dest == nullptr) {
1037     JPEGLI_ERROR("Missing destination.");
1038   }
1039   jpeg_comp_master* m = cinfo->master;
1040   (*cinfo->err->reset_error_mgr)(reinterpret_cast<j_common_ptr>(cinfo));
1041   (*cinfo->dest->init_destination)(cinfo);
1042   jpegli::WriteOutput(cinfo, {0xFF, 0xD8});  // SOI
1043   jpegli::EncodeDQT(cinfo, /*write_all_tables=*/true);
1044   jpegli::CopyHuffmanTables(cinfo);
1045   jpegli::EncodeDHT(cinfo, 0, m->num_huffman_tables);
1046   jpegli::WriteOutput(cinfo, {0xFF, 0xD9});  // EOI
1047   (*cinfo->dest->term_destination)(cinfo);
1048   jpegli_suppress_tables(cinfo, TRUE);
1049 }
1050
1051 //
1052 // Marker writing
1053 //
1054
1055 void jpegli_write_m_header(j_compress_ptr cinfo, int marker,
1056                            unsigned int datalen) {
1057   CheckState(cinfo, jpegli::kEncHeader, jpegli::kEncWriteCoeffs);
1058   if (datalen > jpegli::kMaxBytesInMarker) {
1059     JPEGLI_ERROR("Invalid marker length %u", datalen);
1060   }
1061   if (marker != 0xfe && (marker < 0xe0 || marker > 0xef)) {
1062     JPEGLI_ERROR(
1063         "jpegli_write_m_header: Only APP and COM markers are supported.");
1064   }
1065   std::vector<uint8_t> marker_data(4 + datalen);
1066   marker_data[0] = 0xff;
1067   marker_data[1] = marker;
1068   marker_data[2] = (datalen + 2) >> 8;
1069   marker_data[3] = (datalen + 2) & 0xff;
1070   jpegli::WriteOutput(cinfo, marker_data.data(), 4);
1071 }
1072
1073 void jpegli_write_m_byte(j_compress_ptr cinfo, int val) {
1074   uint8_t data = val;
1075   jpegli::WriteOutput(cinfo, &data, 1);
1076 }
1077
1078 void jpegli_write_marker(j_compress_ptr cinfo, int marker,
1079                          const JOCTET* dataptr, unsigned int datalen) {
1080   jpegli_write_m_header(cinfo, marker, datalen);
1081   jpegli::WriteOutput(cinfo, dataptr, datalen);
1082 }
1083
1084 void jpegli_write_icc_profile(j_compress_ptr cinfo, const JOCTET* icc_data_ptr,
1085                               unsigned int icc_data_len) {
1086   constexpr size_t kMaxIccBytesInMarker =
1087       jpegli::kMaxBytesInMarker - sizeof jpegli::kICCSignature - 2;
1088   const int num_markers =
1089       static_cast<int>(jpegli::DivCeil(icc_data_len, kMaxIccBytesInMarker));
1090   size_t begin = 0;
1091   for (int current_marker = 0; current_marker < num_markers; ++current_marker) {
1092     const size_t length = std::min(kMaxIccBytesInMarker, icc_data_len - begin);
1093     jpegli_write_m_header(
1094         cinfo, jpegli::kICCMarker,
1095         static_cast<unsigned int>(length + sizeof jpegli::kICCSignature + 2));
1096     for (const unsigned char c : jpegli::kICCSignature) {
1097       jpegli_write_m_byte(cinfo, c);
1098     }
1099     jpegli_write_m_byte(cinfo, current_marker + 1);
1100     jpegli_write_m_byte(cinfo, num_markers);
1101     for (size_t i = 0; i < length; ++i) {
1102       jpegli_write_m_byte(cinfo, icc_data_ptr[begin]);
1103       ++begin;
1104     }
1105   }
1106 }
1107
1108 //
1109 // Input streaming
1110 //
1111
1112 JDIMENSION jpegli_write_scanlines(j_compress_ptr cinfo, JSAMPARRAY scanlines,
1113                                   JDIMENSION num_lines) {
1114   CheckState(cinfo, jpegli::kEncHeader, jpegli::kEncReadImage);
1115   if (cinfo->raw_data_in) {
1116     JPEGLI_ERROR("jpegli_write_raw_data() must be called for raw data mode.");
1117   }
1118   jpegli::ProgressMonitorInputPass(cinfo);
1119   if (cinfo->global_state == jpegli::kEncHeader &&
1120       jpegli::IsStreamingSupported(cinfo) && !cinfo->optimize_coding) {
1121     jpegli::WriteFrameHeader(cinfo);
1122     jpegli::WriteScanHeader(cinfo, 0);
1123   }
1124   cinfo->global_state = jpegli::kEncReadImage;
1125   jpeg_comp_master* m = cinfo->master;
1126   if (num_lines + cinfo->next_scanline > cinfo->image_height) {
1127     num_lines = cinfo->image_height - cinfo->next_scanline;
1128   }
1129   JDIMENSION prev_scanline = cinfo->next_scanline;
1130   size_t input_lag = (std::min<size_t>(cinfo->image_height, m->next_input_row) -
1131                       cinfo->next_scanline);
1132   if (input_lag > num_lines) {
1133     JPEGLI_ERROR("Need at least %u lines to continue", input_lag);
1134   }
1135   if (input_lag > 0) {
1136     if (!jpegli::EmptyBitWriterBuffer(&m->bw)) {
1137       return 0;
1138     }
1139     cinfo->next_scanline += input_lag;
1140   }
1141   float* rows[jpegli::kMaxComponents];
1142   for (size_t i = input_lag; i < num_lines; ++i) {
1143     jpegli::ReadInputRow(cinfo, scanlines[i], rows);
1144     (*m->color_transform)(rows, cinfo->image_width);
1145     jpegli::PadInputBuffer(cinfo, rows);
1146     jpegli::ProcessiMCURows(cinfo);
1147     if (!jpegli::EmptyBitWriterBuffer(&m->bw)) {
1148       break;
1149     }
1150     ++cinfo->next_scanline;
1151   }
1152   return cinfo->next_scanline - prev_scanline;
1153 }
1154
1155 JDIMENSION jpegli_write_raw_data(j_compress_ptr cinfo, JSAMPIMAGE data,
1156                                  JDIMENSION num_lines) {
1157   CheckState(cinfo, jpegli::kEncHeader, jpegli::kEncReadImage);
1158   if (!cinfo->raw_data_in) {
1159     JPEGLI_ERROR("jpegli_write_raw_data(): raw data mode was not set");
1160   }
1161   jpegli::ProgressMonitorInputPass(cinfo);
1162   if (cinfo->global_state == jpegli::kEncHeader &&
1163       jpegli::IsStreamingSupported(cinfo) && !cinfo->optimize_coding) {
1164     jpegli::WriteFrameHeader(cinfo);
1165     jpegli::WriteScanHeader(cinfo, 0);
1166   }
1167   cinfo->global_state = jpegli::kEncReadImage;
1168   jpeg_comp_master* m = cinfo->master;
1169   if (cinfo->next_scanline >= cinfo->image_height) {
1170     return 0;
1171   }
1172   size_t iMCU_height = DCTSIZE * cinfo->max_v_samp_factor;
1173   if (num_lines < iMCU_height) {
1174     JPEGLI_ERROR("Missing input lines, minimum is %u", iMCU_height);
1175   }
1176   if (cinfo->next_scanline < m->next_input_row) {
1177     JPEGLI_CHECK(m->next_input_row - cinfo->next_scanline == iMCU_height);
1178     if (!jpegli::EmptyBitWriterBuffer(&m->bw)) {
1179       return 0;
1180     }
1181     cinfo->next_scanline = m->next_input_row;
1182     return iMCU_height;
1183   }
1184   size_t iMCU_y = m->next_input_row / iMCU_height;
1185   float* rows[jpegli::kMaxComponents];
1186   for (int c = 0; c < cinfo->num_components; ++c) {
1187     JSAMPARRAY plane = data[c];
1188     jpeg_component_info* comp = &cinfo->comp_info[c];
1189     size_t xsize = comp->width_in_blocks * DCTSIZE;
1190     size_t ysize = comp->v_samp_factor * DCTSIZE;
1191     size_t y0 = iMCU_y * ysize;
1192     auto& buffer = m->input_buffer[c];
1193     for (size_t i = 0; i < ysize; ++i) {
1194       rows[0] = buffer.Row(y0 + i);
1195       if (plane[i] == nullptr) {
1196         memset(rows[0], 0, xsize * sizeof(rows[0][0]));
1197       } else {
1198         (*m->input_method)(plane[i], xsize, rows);
1199       }
1200       // We need a border of 1 repeated pixel for adaptive quant field.
1201       buffer.PadRow(y0 + i, xsize, /*border=*/1);
1202     }
1203   }
1204   m->next_input_row += iMCU_height;
1205   jpegli::ProcessiMCURows(cinfo);
1206   if (!jpegli::EmptyBitWriterBuffer(&m->bw)) {
1207     return 0;
1208   }
1209   cinfo->next_scanline += iMCU_height;
1210   return iMCU_height;
1211 }
1212
1213 //
1214 // Non-streaming part
1215 //
1216
1217 void jpegli_finish_compress(j_compress_ptr cinfo) {
1218   CheckState(cinfo, jpegli::kEncReadImage, jpegli::kEncWriteCoeffs);
1219   jpeg_comp_master* m = cinfo->master;
1220   if (cinfo->next_scanline < cinfo->image_height) {
1221     JPEGLI_ERROR("Incomplete image, expected %d rows, got %d",
1222                  cinfo->image_height, cinfo->next_scanline);
1223   }
1224
1225   if (cinfo->global_state == jpegli::kEncWriteCoeffs) {
1226     // Zig-zag shuffle all the blocks. For non-transcoding case it was already
1227     // done in EncodeiMCURow().
1228     jpegli::ZigZagShuffleBlocks(cinfo);
1229   }
1230
1231   if (m->psnr_target > 0) {
1232     jpegli::QuantizetoPSNR(cinfo);
1233   }
1234
1235   const bool tokens_done = jpegli::IsStreamingSupported(cinfo);
1236   const bool bitstream_done =
1237       tokens_done && !FROM_JXL_BOOL(cinfo->optimize_coding);
1238
1239   if (!tokens_done) {
1240     jpegli::TokenizeJpeg(cinfo);
1241   }
1242
1243   if (cinfo->optimize_coding || cinfo->progressive_mode) {
1244     jpegli::OptimizeHuffmanCodes(cinfo);
1245     jpegli::InitEntropyCoder(cinfo);
1246   }
1247
1248   if (!bitstream_done) {
1249     jpegli::WriteFrameHeader(cinfo);
1250     for (int i = 0; i < cinfo->num_scans; ++i) {
1251       jpegli::WriteScanHeader(cinfo, i);
1252       jpegli::WriteScanData(cinfo, i);
1253     }
1254   } else {
1255     JumpToByteBoundary(&m->bw);
1256     if (!EmptyBitWriterBuffer(&m->bw)) {
1257       JPEGLI_ERROR("Output suspension is not supported in finish_compress");
1258     }
1259   }
1260
1261   jpegli::WriteOutput(cinfo, {0xFF, 0xD9});  // EOI
1262   (*cinfo->dest->term_destination)(cinfo);
1263
1264   // Release memory and reset global state.
1265   jpegli_abort_compress(cinfo);
1266 }
1267
1268 void jpegli_abort_compress(j_compress_ptr cinfo) {
1269   jpegli_abort(reinterpret_cast<j_common_ptr>(cinfo));
1270 }
1271
1272 void jpegli_destroy_compress(j_compress_ptr cinfo) {
1273   jpegli_destroy(reinterpret_cast<j_common_ptr>(cinfo));
1274 }