third_party/jpeg-xl/lib/jpegli/transpose-inl.h

   1 // Copyright (c) the JPEG XL Project Authors. All rights reserved.
   2 //
   3 // Use of this source code is governed by a BSD-style
   4 // license that can be found in the LICENSE file.
   5
   6 #if defined(LIB_JPEGLI_TRANSPOSE_INL_H_) == defined(HWY_TARGET_TOGGLE)
   7 #ifdef LIB_JPEGLI_TRANSPOSE_INL_H_
   8 #undef LIB_JPEGLI_TRANSPOSE_INL_H_
   9 #else
  10 #define LIB_JPEGLI_TRANSPOSE_INL_H_
  11 #endif
  12
  13 #include "lib/jxl/base/compiler_specific.h"
  14
  15 HWY_BEFORE_NAMESPACE();
  16 namespace jpegli {
  17 namespace HWY_NAMESPACE {
  18 namespace {
  19
  20 #if HWY_CAP_GE256
  21 static JXL_INLINE void Transpose8x8Block(const float* JXL_RESTRICT from,
  22                                          float* JXL_RESTRICT to) {
  23   const HWY_CAPPED(float, 8) d;
  24   auto i0 = Load(d, from);
  25   auto i1 = Load(d, from + 1 * 8);
  26   auto i2 = Load(d, from + 2 * 8);
  27   auto i3 = Load(d, from + 3 * 8);
  28   auto i4 = Load(d, from + 4 * 8);
  29   auto i5 = Load(d, from + 5 * 8);
  30   auto i6 = Load(d, from + 6 * 8);
  31   auto i7 = Load(d, from + 7 * 8);
  32
  33   const auto q0 = InterleaveLower(d, i0, i2);
  34   const auto q1 = InterleaveLower(d, i1, i3);
  35   const auto q2 = InterleaveUpper(d, i0, i2);
  36   const auto q3 = InterleaveUpper(d, i1, i3);
  37   const auto q4 = InterleaveLower(d, i4, i6);
  38   const auto q5 = InterleaveLower(d, i5, i7);
  39   const auto q6 = InterleaveUpper(d, i4, i6);
  40   const auto q7 = InterleaveUpper(d, i5, i7);
  41
  42   const auto r0 = InterleaveLower(d, q0, q1);
  43   const auto r1 = InterleaveUpper(d, q0, q1);
  44   const auto r2 = InterleaveLower(d, q2, q3);
  45   const auto r3 = InterleaveUpper(d, q2, q3);
  46   const auto r4 = InterleaveLower(d, q4, q5);
  47   const auto r5 = InterleaveUpper(d, q4, q5);
  48   const auto r6 = InterleaveLower(d, q6, q7);
  49   const auto r7 = InterleaveUpper(d, q6, q7);
  50
  51   i0 = ConcatLowerLower(d, r4, r0);
  52   i1 = ConcatLowerLower(d, r5, r1);
  53   i2 = ConcatLowerLower(d, r6, r2);
  54   i3 = ConcatLowerLower(d, r7, r3);
  55   i4 = ConcatUpperUpper(d, r4, r0);
  56   i5 = ConcatUpperUpper(d, r5, r1);
  57   i6 = ConcatUpperUpper(d, r6, r2);
  58   i7 = ConcatUpperUpper(d, r7, r3);
  59
  60   Store(i0, d, to);
  61   Store(i1, d, to + 1 * 8);
  62   Store(i2, d, to + 2 * 8);
  63   Store(i3, d, to + 3 * 8);
  64   Store(i4, d, to + 4 * 8);
  65   Store(i5, d, to + 5 * 8);
  66   Store(i6, d, to + 6 * 8);
  67   Store(i7, d, to + 7 * 8);
  68 }
  69 #elif HWY_TARGET != HWY_SCALAR
  70 static JXL_INLINE void Transpose8x8Block(const float* JXL_RESTRICT from,
  71                                          float* JXL_RESTRICT to) {
  72   const HWY_CAPPED(float, 4) d;
  73   for (size_t n = 0; n < 8; n += 4) {
  74     for (size_t m = 0; m < 8; m += 4) {
  75       auto p0 = Load(d, from + n * 8 + m);
  76       auto p1 = Load(d, from + (n + 1) * 8 + m);
  77       auto p2 = Load(d, from + (n + 2) * 8 + m);
  78       auto p3 = Load(d, from + (n + 3) * 8 + m);
  79       const auto q0 = InterleaveLower(d, p0, p2);
  80       const auto q1 = InterleaveLower(d, p1, p3);
  81       const auto q2 = InterleaveUpper(d, p0, p2);
  82       const auto q3 = InterleaveUpper(d, p1, p3);
  83
  84       const auto r0 = InterleaveLower(d, q0, q1);
  85       const auto r1 = InterleaveUpper(d, q0, q1);
  86       const auto r2 = InterleaveLower(d, q2, q3);
  87       const auto r3 = InterleaveUpper(d, q2, q3);
  88       Store(r0, d, to + m * 8 + n);
  89       Store(r1, d, to + (1 + m) * 8 + n);
  90       Store(r2, d, to + (2 + m) * 8 + n);
  91       Store(r3, d, to + (3 + m) * 8 + n);
  92     }
  93   }
  94 }
  95 #else
  96 static JXL_INLINE void Transpose8x8Block(const float* JXL_RESTRICT from,
  97                                          float* JXL_RESTRICT to) {
  98   for (size_t n = 0; n < 8; ++n) {
  99     for (size_t m = 0; m < 8; ++m) {
 100       to[8 * n + m] = from[8 * m + n];
 101     }
 102   }
 103 }
 104 #endif
 105
 106 // NOLINTNEXTLINE(google-readability-namespace-comments)
 107 }  // namespace
 108 }  // namespace HWY_NAMESPACE
 109 }  // namespace jpegli
 110 HWY_AFTER_NAMESPACE();
 111 #endif  // LIB_JPEGLI_TRANSPOSE_INL_H_