2 * Copyright (c) 2014 The WebM project authors. All Rights Reserved.
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
11 #include <emmintrin.h> // SSE2
13 #include "./vpx_dsp_rtcd.h"
14 #include "vp9/encoder/x86/vp9_dct_sse2.h"
15 #include "vpx_dsp/txfm_common.h"
16 #include "vpx_dsp/x86/txfm_common_sse2.h"
17 #include "vpx_ports/mem.h"
19 // TODO(jingning) The high bit-depth functions need rework for performance.
20 // After we properly fix the high bit-depth function implementations, this
21 // file's dependency should be substantially simplified.
22 #if DCT_HIGH_BIT_DEPTH
23 #define ADD_EPI16 _mm_adds_epi16
24 #define SUB_EPI16 _mm_subs_epi16
27 #define ADD_EPI16 _mm_add_epi16
28 #define SUB_EPI16 _mm_sub_epi16
31 void FDCT4x4_2D(const int16_t *input
, tran_low_t
*output
, int stride
) {
32 // This 2D transform implements 4 vertical 1D transforms followed
33 // by 4 horizontal 1D transforms. The multiplies and adds are as given
34 // by Chen, Smith and Fralick ('77). The commands for moving the data
35 // around have been minimized by hand.
36 // For the purposes of the comments, the 16 inputs are referred to at i0
37 // through iF (in raster order), intermediate variables are a0, b0, c0
38 // through f, and correspond to the in-place computations mapped to input
39 // locations. The outputs, o0 through oF are labeled according to the
43 // These are the coefficients used for the multiplies.
44 // In the comments, pN means cos(N pi /64) and mN is -cos(N pi /64),
45 // where cospi_N_64 = cos(N pi /64)
46 const __m128i k__cospi_A
= octa_set_epi16(cospi_16_64
, cospi_16_64
,
47 cospi_16_64
, cospi_16_64
,
48 cospi_16_64
, -cospi_16_64
,
49 cospi_16_64
, -cospi_16_64
);
50 const __m128i k__cospi_B
= octa_set_epi16(cospi_16_64
, -cospi_16_64
,
51 cospi_16_64
, -cospi_16_64
,
52 cospi_16_64
, cospi_16_64
,
53 cospi_16_64
, cospi_16_64
);
54 const __m128i k__cospi_C
= octa_set_epi16(cospi_8_64
, cospi_24_64
,
55 cospi_8_64
, cospi_24_64
,
56 cospi_24_64
, -cospi_8_64
,
57 cospi_24_64
, -cospi_8_64
);
58 const __m128i k__cospi_D
= octa_set_epi16(cospi_24_64
, -cospi_8_64
,
59 cospi_24_64
, -cospi_8_64
,
60 cospi_8_64
, cospi_24_64
,
61 cospi_8_64
, cospi_24_64
);
62 const __m128i k__cospi_E
= octa_set_epi16(cospi_16_64
, cospi_16_64
,
63 cospi_16_64
, cospi_16_64
,
64 cospi_16_64
, cospi_16_64
,
65 cospi_16_64
, cospi_16_64
);
66 const __m128i k__cospi_F
= octa_set_epi16(cospi_16_64
, -cospi_16_64
,
67 cospi_16_64
, -cospi_16_64
,
68 cospi_16_64
, -cospi_16_64
,
69 cospi_16_64
, -cospi_16_64
);
70 const __m128i k__cospi_G
= octa_set_epi16(cospi_8_64
, cospi_24_64
,
71 cospi_8_64
, cospi_24_64
,
72 -cospi_8_64
, -cospi_24_64
,
73 -cospi_8_64
, -cospi_24_64
);
74 const __m128i k__cospi_H
= octa_set_epi16(cospi_24_64
, -cospi_8_64
,
75 cospi_24_64
, -cospi_8_64
,
76 -cospi_24_64
, cospi_8_64
,
77 -cospi_24_64
, cospi_8_64
);
79 const __m128i k__DCT_CONST_ROUNDING
= _mm_set1_epi32(DCT_CONST_ROUNDING
);
80 // This second rounding constant saves doing some extra adds at the end
81 const __m128i k__DCT_CONST_ROUNDING2
= _mm_set1_epi32(DCT_CONST_ROUNDING
82 +(DCT_CONST_ROUNDING
<< 1));
83 const int DCT_CONST_BITS2
= DCT_CONST_BITS
+ 2;
84 const __m128i k__nonzero_bias_a
= _mm_setr_epi16(0, 1, 1, 1, 1, 1, 1, 1);
85 const __m128i k__nonzero_bias_b
= _mm_setr_epi16(1, 0, 0, 0, 0, 0, 0, 0);
87 #if DCT_HIGH_BIT_DEPTH
93 in0
= _mm_loadl_epi64((const __m128i
*)(input
+ 0 * stride
));
94 in1
= _mm_loadl_epi64((const __m128i
*)(input
+ 1 * stride
));
95 in1
= _mm_unpacklo_epi64(in1
, _mm_loadl_epi64((const __m128i
*)
96 (input
+ 2 * stride
)));
97 in0
= _mm_unpacklo_epi64(in0
, _mm_loadl_epi64((const __m128i
*)
98 (input
+ 3 * stride
)));
99 // in0 = [i0 i1 i2 i3 iC iD iE iF]
100 // in1 = [i4 i5 i6 i7 i8 i9 iA iB]
101 #if DCT_HIGH_BIT_DEPTH
102 // Check inputs small enough to use optimised code
103 cmp0
= _mm_xor_si128(_mm_cmpgt_epi16(in0
, _mm_set1_epi16(0x3ff)),
104 _mm_cmplt_epi16(in0
, _mm_set1_epi16(0xfc00)));
105 cmp1
= _mm_xor_si128(_mm_cmpgt_epi16(in1
, _mm_set1_epi16(0x3ff)),
106 _mm_cmplt_epi16(in1
, _mm_set1_epi16(0xfc00)));
107 test
= _mm_movemask_epi8(_mm_or_si128(cmp0
, cmp1
));
109 vp9_highbd_fdct4x4_c(input
, output
, stride
);
112 #endif // DCT_HIGH_BIT_DEPTH
114 // multiply by 16 to give some extra precision
115 in0
= _mm_slli_epi16(in0
, 4);
116 in1
= _mm_slli_epi16(in1
, 4);
117 // if (i == 0 && input[0]) input[0] += 1;
118 // add 1 to the upper left pixel if it is non-zero, which helps reduce
119 // the round-trip error
121 // The mask will only contain whether the first value is zero, all
122 // other comparison will fail as something shifted by 4 (above << 4)
123 // can never be equal to one. To increment in the non-zero case, we
124 // add the mask and one for the first element:
125 // - if zero, mask = -1, v = v - 1 + 1 = v
126 // - if non-zero, mask = 0, v = v + 0 + 1 = v + 1
127 __m128i mask
= _mm_cmpeq_epi16(in0
, k__nonzero_bias_a
);
128 in0
= _mm_add_epi16(in0
, mask
);
129 in0
= _mm_add_epi16(in0
, k__nonzero_bias_b
);
131 // There are 4 total stages, alternating between an add/subtract stage
132 // followed by an multiply-and-add stage.
134 // Stage 1: Add/subtract
136 // in0 = [i0 i1 i2 i3 iC iD iE iF]
137 // in1 = [i4 i5 i6 i7 i8 i9 iA iB]
138 const __m128i r0
= _mm_unpacklo_epi16(in0
, in1
);
139 const __m128i r1
= _mm_unpackhi_epi16(in0
, in1
);
140 // r0 = [i0 i4 i1 i5 i2 i6 i3 i7]
141 // r1 = [iC i8 iD i9 iE iA iF iB]
142 const __m128i r2
= _mm_shuffle_epi32(r0
, 0xB4);
143 const __m128i r3
= _mm_shuffle_epi32(r1
, 0xB4);
144 // r2 = [i0 i4 i1 i5 i3 i7 i2 i6]
145 // r3 = [iC i8 iD i9 iF iB iE iA]
147 const __m128i t0
= _mm_add_epi16(r2
, r3
);
148 const __m128i t1
= _mm_sub_epi16(r2
, r3
);
149 // t0 = [a0 a4 a1 a5 a3 a7 a2 a6]
150 // t1 = [aC a8 aD a9 aF aB aE aA]
152 // Stage 2: multiply by constants (which gets us into 32 bits).
153 // The constants needed here are:
154 // k__cospi_A = [p16 p16 p16 p16 p16 m16 p16 m16]
155 // k__cospi_B = [p16 m16 p16 m16 p16 p16 p16 p16]
156 // k__cospi_C = [p08 p24 p08 p24 p24 m08 p24 m08]
157 // k__cospi_D = [p24 m08 p24 m08 p08 p24 p08 p24]
158 const __m128i u0
= _mm_madd_epi16(t0
, k__cospi_A
);
159 const __m128i u2
= _mm_madd_epi16(t0
, k__cospi_B
);
160 const __m128i u1
= _mm_madd_epi16(t1
, k__cospi_C
);
161 const __m128i u3
= _mm_madd_epi16(t1
, k__cospi_D
);
162 // Then add and right-shift to get back to 16-bit range
163 const __m128i v0
= _mm_add_epi32(u0
, k__DCT_CONST_ROUNDING
);
164 const __m128i v1
= _mm_add_epi32(u1
, k__DCT_CONST_ROUNDING
);
165 const __m128i v2
= _mm_add_epi32(u2
, k__DCT_CONST_ROUNDING
);
166 const __m128i v3
= _mm_add_epi32(u3
, k__DCT_CONST_ROUNDING
);
167 const __m128i w0
= _mm_srai_epi32(v0
, DCT_CONST_BITS
);
168 const __m128i w1
= _mm_srai_epi32(v1
, DCT_CONST_BITS
);
169 const __m128i w2
= _mm_srai_epi32(v2
, DCT_CONST_BITS
);
170 const __m128i w3
= _mm_srai_epi32(v3
, DCT_CONST_BITS
);
171 // w0 = [b0 b1 b7 b6]
172 // w1 = [b8 b9 bF bE]
173 // w2 = [b4 b5 b3 b2]
174 // w3 = [bC bD bB bA]
175 const __m128i x0
= _mm_packs_epi32(w0
, w1
);
176 const __m128i x1
= _mm_packs_epi32(w2
, w3
);
177 #if DCT_HIGH_BIT_DEPTH
178 overflow
= check_epi16_overflow_x2(&x0
, &x1
);
180 vp9_highbd_fdct4x4_c(input
, output
, stride
);
183 #endif // DCT_HIGH_BIT_DEPTH
184 // x0 = [b0 b1 b7 b6 b8 b9 bF bE]
185 // x1 = [b4 b5 b3 b2 bC bD bB bA]
186 in0
= _mm_shuffle_epi32(x0
, 0xD8);
187 in1
= _mm_shuffle_epi32(x1
, 0x8D);
188 // in0 = [b0 b1 b8 b9 b7 b6 bF bE]
189 // in1 = [b3 b2 bB bA b4 b5 bC bD]
192 // vertical DCTs finished. Now we do the horizontal DCTs.
193 // Stage 3: Add/subtract
195 const __m128i t0
= ADD_EPI16(in0
, in1
);
196 const __m128i t1
= SUB_EPI16(in0
, in1
);
197 // t0 = [c0 c1 c8 c9 c4 c5 cC cD]
198 // t1 = [c3 c2 cB cA -c7 -c6 -cF -cE]
199 #if DCT_HIGH_BIT_DEPTH
200 overflow
= check_epi16_overflow_x2(&t0
, &t1
);
202 vp9_highbd_fdct4x4_c(input
, output
, stride
);
205 #endif // DCT_HIGH_BIT_DEPTH
207 // Stage 4: multiply by constants (which gets us into 32 bits).
209 // The constants needed here are:
210 // k__cospi_E = [p16 p16 p16 p16 p16 p16 p16 p16]
211 // k__cospi_F = [p16 m16 p16 m16 p16 m16 p16 m16]
212 // k__cospi_G = [p08 p24 p08 p24 m08 m24 m08 m24]
213 // k__cospi_H = [p24 m08 p24 m08 m24 p08 m24 p08]
214 const __m128i u0
= _mm_madd_epi16(t0
, k__cospi_E
);
215 const __m128i u1
= _mm_madd_epi16(t0
, k__cospi_F
);
216 const __m128i u2
= _mm_madd_epi16(t1
, k__cospi_G
);
217 const __m128i u3
= _mm_madd_epi16(t1
, k__cospi_H
);
218 // Then add and right-shift to get back to 16-bit range
219 // but this combines the final right-shift as well to save operations
220 // This unusual rounding operations is to maintain bit-accurate
221 // compatibility with the c version of this function which has two
222 // rounding steps in a row.
223 const __m128i v0
= _mm_add_epi32(u0
, k__DCT_CONST_ROUNDING2
);
224 const __m128i v1
= _mm_add_epi32(u1
, k__DCT_CONST_ROUNDING2
);
225 const __m128i v2
= _mm_add_epi32(u2
, k__DCT_CONST_ROUNDING2
);
226 const __m128i v3
= _mm_add_epi32(u3
, k__DCT_CONST_ROUNDING2
);
227 const __m128i w0
= _mm_srai_epi32(v0
, DCT_CONST_BITS2
);
228 const __m128i w1
= _mm_srai_epi32(v1
, DCT_CONST_BITS2
);
229 const __m128i w2
= _mm_srai_epi32(v2
, DCT_CONST_BITS2
);
230 const __m128i w3
= _mm_srai_epi32(v3
, DCT_CONST_BITS2
);
231 // w0 = [o0 o4 o8 oC]
232 // w1 = [o2 o6 oA oE]
233 // w2 = [o1 o5 o9 oD]
234 // w3 = [o3 o7 oB oF]
235 // remember the o's are numbered according to the correct output location
236 const __m128i x0
= _mm_packs_epi32(w0
, w1
);
237 const __m128i x1
= _mm_packs_epi32(w2
, w3
);
238 #if DCT_HIGH_BIT_DEPTH
239 overflow
= check_epi16_overflow_x2(&x0
, &x1
);
241 vp9_highbd_fdct4x4_c(input
, output
, stride
);
244 #endif // DCT_HIGH_BIT_DEPTH
246 // x0 = [o0 o4 o8 oC o2 o6 oA oE]
247 // x1 = [o1 o5 o9 oD o3 o7 oB oF]
248 const __m128i y0
= _mm_unpacklo_epi16(x0
, x1
);
249 const __m128i y1
= _mm_unpackhi_epi16(x0
, x1
);
250 // y0 = [o0 o1 o4 o5 o8 o9 oC oD]
251 // y1 = [o2 o3 o6 o7 oA oB oE oF]
252 in0
= _mm_unpacklo_epi32(y0
, y1
);
253 // in0 = [o0 o1 o2 o3 o4 o5 o6 o7]
254 in1
= _mm_unpackhi_epi32(y0
, y1
);
255 // in1 = [o8 o9 oA oB oC oD oE oF]
259 // Post-condition (v + 1) >> 2 is now incorporated into previous
260 // add and right-shift commands. Only 2 store instructions needed
261 // because we are using the fact that 1/3 are stored just after 0/2.
262 storeu_output(&in0
, output
+ 0 * 4);
263 storeu_output(&in1
, output
+ 2 * 4);
267 void FDCT8x8_2D(const int16_t *input
, tran_low_t
*output
, int stride
) {
270 // When we use them, in one case, they are all the same. In all others
271 // it's a pair of them that we need to repeat four times. This is done
272 // by constructing the 32 bit constant corresponding to that pair.
273 const __m128i k__cospi_p16_p16
= _mm_set1_epi16((int16_t)cospi_16_64
);
274 const __m128i k__cospi_p16_m16
= pair_set_epi16(cospi_16_64
, -cospi_16_64
);
275 const __m128i k__cospi_p24_p08
= pair_set_epi16(cospi_24_64
, cospi_8_64
);
276 const __m128i k__cospi_m08_p24
= pair_set_epi16(-cospi_8_64
, cospi_24_64
);
277 const __m128i k__cospi_p28_p04
= pair_set_epi16(cospi_28_64
, cospi_4_64
);
278 const __m128i k__cospi_m04_p28
= pair_set_epi16(-cospi_4_64
, cospi_28_64
);
279 const __m128i k__cospi_p12_p20
= pair_set_epi16(cospi_12_64
, cospi_20_64
);
280 const __m128i k__cospi_m20_p12
= pair_set_epi16(-cospi_20_64
, cospi_12_64
);
281 const __m128i k__DCT_CONST_ROUNDING
= _mm_set1_epi32(DCT_CONST_ROUNDING
);
282 #if DCT_HIGH_BIT_DEPTH
286 __m128i in0
= _mm_load_si128((const __m128i
*)(input
+ 0 * stride
));
287 __m128i in1
= _mm_load_si128((const __m128i
*)(input
+ 1 * stride
));
288 __m128i in2
= _mm_load_si128((const __m128i
*)(input
+ 2 * stride
));
289 __m128i in3
= _mm_load_si128((const __m128i
*)(input
+ 3 * stride
));
290 __m128i in4
= _mm_load_si128((const __m128i
*)(input
+ 4 * stride
));
291 __m128i in5
= _mm_load_si128((const __m128i
*)(input
+ 5 * stride
));
292 __m128i in6
= _mm_load_si128((const __m128i
*)(input
+ 6 * stride
));
293 __m128i in7
= _mm_load_si128((const __m128i
*)(input
+ 7 * stride
));
294 // Pre-condition input (shift by two)
295 in0
= _mm_slli_epi16(in0
, 2);
296 in1
= _mm_slli_epi16(in1
, 2);
297 in2
= _mm_slli_epi16(in2
, 2);
298 in3
= _mm_slli_epi16(in3
, 2);
299 in4
= _mm_slli_epi16(in4
, 2);
300 in5
= _mm_slli_epi16(in5
, 2);
301 in6
= _mm_slli_epi16(in6
, 2);
302 in7
= _mm_slli_epi16(in7
, 2);
304 // We do two passes, first the columns, then the rows. The results of the
305 // first pass are transposed so that the same column code can be reused. The
306 // results of the second pass are also transposed so that the rows (processed
307 // as columns) are put back in row positions.
308 for (pass
= 0; pass
< 2; pass
++) {
309 // To store results of each pass before the transpose.
310 __m128i res0
, res1
, res2
, res3
, res4
, res5
, res6
, res7
;
312 const __m128i q0
= ADD_EPI16(in0
, in7
);
313 const __m128i q1
= ADD_EPI16(in1
, in6
);
314 const __m128i q2
= ADD_EPI16(in2
, in5
);
315 const __m128i q3
= ADD_EPI16(in3
, in4
);
316 const __m128i q4
= SUB_EPI16(in3
, in4
);
317 const __m128i q5
= SUB_EPI16(in2
, in5
);
318 const __m128i q6
= SUB_EPI16(in1
, in6
);
319 const __m128i q7
= SUB_EPI16(in0
, in7
);
320 #if DCT_HIGH_BIT_DEPTH
322 overflow
= check_epi16_overflow_x8(&q0
, &q1
, &q2
, &q3
,
325 vp9_highbd_fdct8x8_c(input
, output
, stride
);
329 #endif // DCT_HIGH_BIT_DEPTH
330 // Work on first four results
333 const __m128i r0
= ADD_EPI16(q0
, q3
);
334 const __m128i r1
= ADD_EPI16(q1
, q2
);
335 const __m128i r2
= SUB_EPI16(q1
, q2
);
336 const __m128i r3
= SUB_EPI16(q0
, q3
);
337 #if DCT_HIGH_BIT_DEPTH
338 overflow
= check_epi16_overflow_x4(&r0
, &r1
, &r2
, &r3
);
340 vp9_highbd_fdct8x8_c(input
, output
, stride
);
343 #endif // DCT_HIGH_BIT_DEPTH
344 // Interleave to do the multiply by constants which gets us into 32bits
346 const __m128i t0
= _mm_unpacklo_epi16(r0
, r1
);
347 const __m128i t1
= _mm_unpackhi_epi16(r0
, r1
);
348 const __m128i t2
= _mm_unpacklo_epi16(r2
, r3
);
349 const __m128i t3
= _mm_unpackhi_epi16(r2
, r3
);
350 const __m128i u0
= _mm_madd_epi16(t0
, k__cospi_p16_p16
);
351 const __m128i u1
= _mm_madd_epi16(t1
, k__cospi_p16_p16
);
352 const __m128i u2
= _mm_madd_epi16(t0
, k__cospi_p16_m16
);
353 const __m128i u3
= _mm_madd_epi16(t1
, k__cospi_p16_m16
);
354 const __m128i u4
= _mm_madd_epi16(t2
, k__cospi_p24_p08
);
355 const __m128i u5
= _mm_madd_epi16(t3
, k__cospi_p24_p08
);
356 const __m128i u6
= _mm_madd_epi16(t2
, k__cospi_m08_p24
);
357 const __m128i u7
= _mm_madd_epi16(t3
, k__cospi_m08_p24
);
358 // dct_const_round_shift
359 const __m128i v0
= _mm_add_epi32(u0
, k__DCT_CONST_ROUNDING
);
360 const __m128i v1
= _mm_add_epi32(u1
, k__DCT_CONST_ROUNDING
);
361 const __m128i v2
= _mm_add_epi32(u2
, k__DCT_CONST_ROUNDING
);
362 const __m128i v3
= _mm_add_epi32(u3
, k__DCT_CONST_ROUNDING
);
363 const __m128i v4
= _mm_add_epi32(u4
, k__DCT_CONST_ROUNDING
);
364 const __m128i v5
= _mm_add_epi32(u5
, k__DCT_CONST_ROUNDING
);
365 const __m128i v6
= _mm_add_epi32(u6
, k__DCT_CONST_ROUNDING
);
366 const __m128i v7
= _mm_add_epi32(u7
, k__DCT_CONST_ROUNDING
);
367 const __m128i w0
= _mm_srai_epi32(v0
, DCT_CONST_BITS
);
368 const __m128i w1
= _mm_srai_epi32(v1
, DCT_CONST_BITS
);
369 const __m128i w2
= _mm_srai_epi32(v2
, DCT_CONST_BITS
);
370 const __m128i w3
= _mm_srai_epi32(v3
, DCT_CONST_BITS
);
371 const __m128i w4
= _mm_srai_epi32(v4
, DCT_CONST_BITS
);
372 const __m128i w5
= _mm_srai_epi32(v5
, DCT_CONST_BITS
);
373 const __m128i w6
= _mm_srai_epi32(v6
, DCT_CONST_BITS
);
374 const __m128i w7
= _mm_srai_epi32(v7
, DCT_CONST_BITS
);
376 res0
= _mm_packs_epi32(w0
, w1
);
377 res4
= _mm_packs_epi32(w2
, w3
);
378 res2
= _mm_packs_epi32(w4
, w5
);
379 res6
= _mm_packs_epi32(w6
, w7
);
380 #if DCT_HIGH_BIT_DEPTH
381 overflow
= check_epi16_overflow_x4(&res0
, &res4
, &res2
, &res6
);
383 vp9_highbd_fdct8x8_c(input
, output
, stride
);
386 #endif // DCT_HIGH_BIT_DEPTH
389 // Work on next four results
391 // Interleave to do the multiply by constants which gets us into 32bits
392 const __m128i d0
= _mm_unpacklo_epi16(q6
, q5
);
393 const __m128i d1
= _mm_unpackhi_epi16(q6
, q5
);
394 const __m128i e0
= _mm_madd_epi16(d0
, k__cospi_p16_m16
);
395 const __m128i e1
= _mm_madd_epi16(d1
, k__cospi_p16_m16
);
396 const __m128i e2
= _mm_madd_epi16(d0
, k__cospi_p16_p16
);
397 const __m128i e3
= _mm_madd_epi16(d1
, k__cospi_p16_p16
);
398 // dct_const_round_shift
399 const __m128i f0
= _mm_add_epi32(e0
, k__DCT_CONST_ROUNDING
);
400 const __m128i f1
= _mm_add_epi32(e1
, k__DCT_CONST_ROUNDING
);
401 const __m128i f2
= _mm_add_epi32(e2
, k__DCT_CONST_ROUNDING
);
402 const __m128i f3
= _mm_add_epi32(e3
, k__DCT_CONST_ROUNDING
);
403 const __m128i s0
= _mm_srai_epi32(f0
, DCT_CONST_BITS
);
404 const __m128i s1
= _mm_srai_epi32(f1
, DCT_CONST_BITS
);
405 const __m128i s2
= _mm_srai_epi32(f2
, DCT_CONST_BITS
);
406 const __m128i s3
= _mm_srai_epi32(f3
, DCT_CONST_BITS
);
408 const __m128i r0
= _mm_packs_epi32(s0
, s1
);
409 const __m128i r1
= _mm_packs_epi32(s2
, s3
);
410 #if DCT_HIGH_BIT_DEPTH
411 overflow
= check_epi16_overflow_x2(&r0
, &r1
);
413 vp9_highbd_fdct8x8_c(input
, output
, stride
);
416 #endif // DCT_HIGH_BIT_DEPTH
419 const __m128i x0
= ADD_EPI16(q4
, r0
);
420 const __m128i x1
= SUB_EPI16(q4
, r0
);
421 const __m128i x2
= SUB_EPI16(q7
, r1
);
422 const __m128i x3
= ADD_EPI16(q7
, r1
);
423 #if DCT_HIGH_BIT_DEPTH
424 overflow
= check_epi16_overflow_x4(&x0
, &x1
, &x2
, &x3
);
426 vp9_highbd_fdct8x8_c(input
, output
, stride
);
429 #endif // DCT_HIGH_BIT_DEPTH
430 // Interleave to do the multiply by constants which gets us into 32bits
432 const __m128i t0
= _mm_unpacklo_epi16(x0
, x3
);
433 const __m128i t1
= _mm_unpackhi_epi16(x0
, x3
);
434 const __m128i t2
= _mm_unpacklo_epi16(x1
, x2
);
435 const __m128i t3
= _mm_unpackhi_epi16(x1
, x2
);
436 const __m128i u0
= _mm_madd_epi16(t0
, k__cospi_p28_p04
);
437 const __m128i u1
= _mm_madd_epi16(t1
, k__cospi_p28_p04
);
438 const __m128i u2
= _mm_madd_epi16(t0
, k__cospi_m04_p28
);
439 const __m128i u3
= _mm_madd_epi16(t1
, k__cospi_m04_p28
);
440 const __m128i u4
= _mm_madd_epi16(t2
, k__cospi_p12_p20
);
441 const __m128i u5
= _mm_madd_epi16(t3
, k__cospi_p12_p20
);
442 const __m128i u6
= _mm_madd_epi16(t2
, k__cospi_m20_p12
);
443 const __m128i u7
= _mm_madd_epi16(t3
, k__cospi_m20_p12
);
444 // dct_const_round_shift
445 const __m128i v0
= _mm_add_epi32(u0
, k__DCT_CONST_ROUNDING
);
446 const __m128i v1
= _mm_add_epi32(u1
, k__DCT_CONST_ROUNDING
);
447 const __m128i v2
= _mm_add_epi32(u2
, k__DCT_CONST_ROUNDING
);
448 const __m128i v3
= _mm_add_epi32(u3
, k__DCT_CONST_ROUNDING
);
449 const __m128i v4
= _mm_add_epi32(u4
, k__DCT_CONST_ROUNDING
);
450 const __m128i v5
= _mm_add_epi32(u5
, k__DCT_CONST_ROUNDING
);
451 const __m128i v6
= _mm_add_epi32(u6
, k__DCT_CONST_ROUNDING
);
452 const __m128i v7
= _mm_add_epi32(u7
, k__DCT_CONST_ROUNDING
);
453 const __m128i w0
= _mm_srai_epi32(v0
, DCT_CONST_BITS
);
454 const __m128i w1
= _mm_srai_epi32(v1
, DCT_CONST_BITS
);
455 const __m128i w2
= _mm_srai_epi32(v2
, DCT_CONST_BITS
);
456 const __m128i w3
= _mm_srai_epi32(v3
, DCT_CONST_BITS
);
457 const __m128i w4
= _mm_srai_epi32(v4
, DCT_CONST_BITS
);
458 const __m128i w5
= _mm_srai_epi32(v5
, DCT_CONST_BITS
);
459 const __m128i w6
= _mm_srai_epi32(v6
, DCT_CONST_BITS
);
460 const __m128i w7
= _mm_srai_epi32(v7
, DCT_CONST_BITS
);
462 res1
= _mm_packs_epi32(w0
, w1
);
463 res7
= _mm_packs_epi32(w2
, w3
);
464 res5
= _mm_packs_epi32(w4
, w5
);
465 res3
= _mm_packs_epi32(w6
, w7
);
466 #if DCT_HIGH_BIT_DEPTH
467 overflow
= check_epi16_overflow_x4(&res1
, &res7
, &res5
, &res3
);
469 vp9_highbd_fdct8x8_c(input
, output
, stride
);
472 #endif // DCT_HIGH_BIT_DEPTH
476 // Transpose the 8x8.
478 // 00 01 02 03 04 05 06 07
479 // 10 11 12 13 14 15 16 17
480 // 20 21 22 23 24 25 26 27
481 // 30 31 32 33 34 35 36 37
482 // 40 41 42 43 44 45 46 47
483 // 50 51 52 53 54 55 56 57
484 // 60 61 62 63 64 65 66 67
485 // 70 71 72 73 74 75 76 77
486 const __m128i tr0_0
= _mm_unpacklo_epi16(res0
, res1
);
487 const __m128i tr0_1
= _mm_unpacklo_epi16(res2
, res3
);
488 const __m128i tr0_2
= _mm_unpackhi_epi16(res0
, res1
);
489 const __m128i tr0_3
= _mm_unpackhi_epi16(res2
, res3
);
490 const __m128i tr0_4
= _mm_unpacklo_epi16(res4
, res5
);
491 const __m128i tr0_5
= _mm_unpacklo_epi16(res6
, res7
);
492 const __m128i tr0_6
= _mm_unpackhi_epi16(res4
, res5
);
493 const __m128i tr0_7
= _mm_unpackhi_epi16(res6
, res7
);
494 // 00 10 01 11 02 12 03 13
495 // 20 30 21 31 22 32 23 33
496 // 04 14 05 15 06 16 07 17
497 // 24 34 25 35 26 36 27 37
498 // 40 50 41 51 42 52 43 53
499 // 60 70 61 71 62 72 63 73
500 // 54 54 55 55 56 56 57 57
501 // 64 74 65 75 66 76 67 77
502 const __m128i tr1_0
= _mm_unpacklo_epi32(tr0_0
, tr0_1
);
503 const __m128i tr1_1
= _mm_unpacklo_epi32(tr0_2
, tr0_3
);
504 const __m128i tr1_2
= _mm_unpackhi_epi32(tr0_0
, tr0_1
);
505 const __m128i tr1_3
= _mm_unpackhi_epi32(tr0_2
, tr0_3
);
506 const __m128i tr1_4
= _mm_unpacklo_epi32(tr0_4
, tr0_5
);
507 const __m128i tr1_5
= _mm_unpacklo_epi32(tr0_6
, tr0_7
);
508 const __m128i tr1_6
= _mm_unpackhi_epi32(tr0_4
, tr0_5
);
509 const __m128i tr1_7
= _mm_unpackhi_epi32(tr0_6
, tr0_7
);
510 // 00 10 20 30 01 11 21 31
511 // 40 50 60 70 41 51 61 71
512 // 02 12 22 32 03 13 23 33
513 // 42 52 62 72 43 53 63 73
514 // 04 14 24 34 05 15 21 36
515 // 44 54 64 74 45 55 61 76
516 // 06 16 26 36 07 17 27 37
517 // 46 56 66 76 47 57 67 77
518 in0
= _mm_unpacklo_epi64(tr1_0
, tr1_4
);
519 in1
= _mm_unpackhi_epi64(tr1_0
, tr1_4
);
520 in2
= _mm_unpacklo_epi64(tr1_2
, tr1_6
);
521 in3
= _mm_unpackhi_epi64(tr1_2
, tr1_6
);
522 in4
= _mm_unpacklo_epi64(tr1_1
, tr1_5
);
523 in5
= _mm_unpackhi_epi64(tr1_1
, tr1_5
);
524 in6
= _mm_unpacklo_epi64(tr1_3
, tr1_7
);
525 in7
= _mm_unpackhi_epi64(tr1_3
, tr1_7
);
526 // 00 10 20 30 40 50 60 70
527 // 01 11 21 31 41 51 61 71
528 // 02 12 22 32 42 52 62 72
529 // 03 13 23 33 43 53 63 73
530 // 04 14 24 34 44 54 64 74
531 // 05 15 25 35 45 55 65 75
532 // 06 16 26 36 46 56 66 76
533 // 07 17 27 37 47 57 67 77
536 // Post-condition output and store it
538 // Post-condition (division by two)
539 // division of two 16 bits signed numbers using shifts
540 // n / 2 = (n - (n >> 15)) >> 1
541 const __m128i sign_in0
= _mm_srai_epi16(in0
, 15);
542 const __m128i sign_in1
= _mm_srai_epi16(in1
, 15);
543 const __m128i sign_in2
= _mm_srai_epi16(in2
, 15);
544 const __m128i sign_in3
= _mm_srai_epi16(in3
, 15);
545 const __m128i sign_in4
= _mm_srai_epi16(in4
, 15);
546 const __m128i sign_in5
= _mm_srai_epi16(in5
, 15);
547 const __m128i sign_in6
= _mm_srai_epi16(in6
, 15);
548 const __m128i sign_in7
= _mm_srai_epi16(in7
, 15);
549 in0
= _mm_sub_epi16(in0
, sign_in0
);
550 in1
= _mm_sub_epi16(in1
, sign_in1
);
551 in2
= _mm_sub_epi16(in2
, sign_in2
);
552 in3
= _mm_sub_epi16(in3
, sign_in3
);
553 in4
= _mm_sub_epi16(in4
, sign_in4
);
554 in5
= _mm_sub_epi16(in5
, sign_in5
);
555 in6
= _mm_sub_epi16(in6
, sign_in6
);
556 in7
= _mm_sub_epi16(in7
, sign_in7
);
557 in0
= _mm_srai_epi16(in0
, 1);
558 in1
= _mm_srai_epi16(in1
, 1);
559 in2
= _mm_srai_epi16(in2
, 1);
560 in3
= _mm_srai_epi16(in3
, 1);
561 in4
= _mm_srai_epi16(in4
, 1);
562 in5
= _mm_srai_epi16(in5
, 1);
563 in6
= _mm_srai_epi16(in6
, 1);
564 in7
= _mm_srai_epi16(in7
, 1);
566 store_output(&in0
, (output
+ 0 * 8));
567 store_output(&in1
, (output
+ 1 * 8));
568 store_output(&in2
, (output
+ 2 * 8));
569 store_output(&in3
, (output
+ 3 * 8));
570 store_output(&in4
, (output
+ 4 * 8));
571 store_output(&in5
, (output
+ 5 * 8));
572 store_output(&in6
, (output
+ 6 * 8));
573 store_output(&in7
, (output
+ 7 * 8));
577 void FDCT16x16_2D(const int16_t *input
, tran_low_t
*output
, int stride
) {
578 // The 2D transform is done with two passes which are actually pretty
579 // similar. In the first one, we transform the columns and transpose
580 // the results. In the second one, we transform the rows. To achieve that,
581 // as the first pass results are transposed, we transpose the columns (that
582 // is the transposed rows) and transpose the results (so that it goes back
583 // in normal/row positions).
585 // We need an intermediate buffer between passes.
586 DECLARE_ALIGNED(16, int16_t, intermediate
[256]);
587 const int16_t *in
= input
;
588 int16_t *out0
= intermediate
;
589 tran_low_t
*out1
= output
;
591 // When we use them, in one case, they are all the same. In all others
592 // it's a pair of them that we need to repeat four times. This is done
593 // by constructing the 32 bit constant corresponding to that pair.
594 const __m128i k__cospi_p16_p16
= _mm_set1_epi16((int16_t)cospi_16_64
);
595 const __m128i k__cospi_p16_m16
= pair_set_epi16(cospi_16_64
, -cospi_16_64
);
596 const __m128i k__cospi_p24_p08
= pair_set_epi16(cospi_24_64
, cospi_8_64
);
597 const __m128i k__cospi_p08_m24
= pair_set_epi16(cospi_8_64
, -cospi_24_64
);
598 const __m128i k__cospi_m08_p24
= pair_set_epi16(-cospi_8_64
, cospi_24_64
);
599 const __m128i k__cospi_p28_p04
= pair_set_epi16(cospi_28_64
, cospi_4_64
);
600 const __m128i k__cospi_m04_p28
= pair_set_epi16(-cospi_4_64
, cospi_28_64
);
601 const __m128i k__cospi_p12_p20
= pair_set_epi16(cospi_12_64
, cospi_20_64
);
602 const __m128i k__cospi_m20_p12
= pair_set_epi16(-cospi_20_64
, cospi_12_64
);
603 const __m128i k__cospi_p30_p02
= pair_set_epi16(cospi_30_64
, cospi_2_64
);
604 const __m128i k__cospi_p14_p18
= pair_set_epi16(cospi_14_64
, cospi_18_64
);
605 const __m128i k__cospi_m02_p30
= pair_set_epi16(-cospi_2_64
, cospi_30_64
);
606 const __m128i k__cospi_m18_p14
= pair_set_epi16(-cospi_18_64
, cospi_14_64
);
607 const __m128i k__cospi_p22_p10
= pair_set_epi16(cospi_22_64
, cospi_10_64
);
608 const __m128i k__cospi_p06_p26
= pair_set_epi16(cospi_6_64
, cospi_26_64
);
609 const __m128i k__cospi_m10_p22
= pair_set_epi16(-cospi_10_64
, cospi_22_64
);
610 const __m128i k__cospi_m26_p06
= pair_set_epi16(-cospi_26_64
, cospi_6_64
);
611 const __m128i k__DCT_CONST_ROUNDING
= _mm_set1_epi32(DCT_CONST_ROUNDING
);
612 const __m128i kOne
= _mm_set1_epi16(1);
613 // Do the two transform/transpose passes
614 for (pass
= 0; pass
< 2; ++pass
) {
615 // We process eight columns (transposed rows in second pass) at a time.
617 #if DCT_HIGH_BIT_DEPTH
620 for (column_start
= 0; column_start
< 16; column_start
+= 8) {
621 __m128i in00
, in01
, in02
, in03
, in04
, in05
, in06
, in07
;
622 __m128i in08
, in09
, in10
, in11
, in12
, in13
, in14
, in15
;
623 __m128i input0
, input1
, input2
, input3
, input4
, input5
, input6
, input7
;
624 __m128i step1_0
, step1_1
, step1_2
, step1_3
;
625 __m128i step1_4
, step1_5
, step1_6
, step1_7
;
626 __m128i step2_1
, step2_2
, step2_3
, step2_4
, step2_5
, step2_6
;
627 __m128i step3_0
, step3_1
, step3_2
, step3_3
;
628 __m128i step3_4
, step3_5
, step3_6
, step3_7
;
629 __m128i res00
, res01
, res02
, res03
, res04
, res05
, res06
, res07
;
630 __m128i res08
, res09
, res10
, res11
, res12
, res13
, res14
, res15
;
631 // Load and pre-condition input.
633 in00
= _mm_load_si128((const __m128i
*)(in
+ 0 * stride
));
634 in01
= _mm_load_si128((const __m128i
*)(in
+ 1 * stride
));
635 in02
= _mm_load_si128((const __m128i
*)(in
+ 2 * stride
));
636 in03
= _mm_load_si128((const __m128i
*)(in
+ 3 * stride
));
637 in04
= _mm_load_si128((const __m128i
*)(in
+ 4 * stride
));
638 in05
= _mm_load_si128((const __m128i
*)(in
+ 5 * stride
));
639 in06
= _mm_load_si128((const __m128i
*)(in
+ 6 * stride
));
640 in07
= _mm_load_si128((const __m128i
*)(in
+ 7 * stride
));
641 in08
= _mm_load_si128((const __m128i
*)(in
+ 8 * stride
));
642 in09
= _mm_load_si128((const __m128i
*)(in
+ 9 * stride
));
643 in10
= _mm_load_si128((const __m128i
*)(in
+ 10 * stride
));
644 in11
= _mm_load_si128((const __m128i
*)(in
+ 11 * stride
));
645 in12
= _mm_load_si128((const __m128i
*)(in
+ 12 * stride
));
646 in13
= _mm_load_si128((const __m128i
*)(in
+ 13 * stride
));
647 in14
= _mm_load_si128((const __m128i
*)(in
+ 14 * stride
));
648 in15
= _mm_load_si128((const __m128i
*)(in
+ 15 * stride
));
650 in00
= _mm_slli_epi16(in00
, 2);
651 in01
= _mm_slli_epi16(in01
, 2);
652 in02
= _mm_slli_epi16(in02
, 2);
653 in03
= _mm_slli_epi16(in03
, 2);
654 in04
= _mm_slli_epi16(in04
, 2);
655 in05
= _mm_slli_epi16(in05
, 2);
656 in06
= _mm_slli_epi16(in06
, 2);
657 in07
= _mm_slli_epi16(in07
, 2);
658 in08
= _mm_slli_epi16(in08
, 2);
659 in09
= _mm_slli_epi16(in09
, 2);
660 in10
= _mm_slli_epi16(in10
, 2);
661 in11
= _mm_slli_epi16(in11
, 2);
662 in12
= _mm_slli_epi16(in12
, 2);
663 in13
= _mm_slli_epi16(in13
, 2);
664 in14
= _mm_slli_epi16(in14
, 2);
665 in15
= _mm_slli_epi16(in15
, 2);
667 in00
= _mm_load_si128((const __m128i
*)(in
+ 0 * 16));
668 in01
= _mm_load_si128((const __m128i
*)(in
+ 1 * 16));
669 in02
= _mm_load_si128((const __m128i
*)(in
+ 2 * 16));
670 in03
= _mm_load_si128((const __m128i
*)(in
+ 3 * 16));
671 in04
= _mm_load_si128((const __m128i
*)(in
+ 4 * 16));
672 in05
= _mm_load_si128((const __m128i
*)(in
+ 5 * 16));
673 in06
= _mm_load_si128((const __m128i
*)(in
+ 6 * 16));
674 in07
= _mm_load_si128((const __m128i
*)(in
+ 7 * 16));
675 in08
= _mm_load_si128((const __m128i
*)(in
+ 8 * 16));
676 in09
= _mm_load_si128((const __m128i
*)(in
+ 9 * 16));
677 in10
= _mm_load_si128((const __m128i
*)(in
+ 10 * 16));
678 in11
= _mm_load_si128((const __m128i
*)(in
+ 11 * 16));
679 in12
= _mm_load_si128((const __m128i
*)(in
+ 12 * 16));
680 in13
= _mm_load_si128((const __m128i
*)(in
+ 13 * 16));
681 in14
= _mm_load_si128((const __m128i
*)(in
+ 14 * 16));
682 in15
= _mm_load_si128((const __m128i
*)(in
+ 15 * 16));
684 in00
= _mm_add_epi16(in00
, kOne
);
685 in01
= _mm_add_epi16(in01
, kOne
);
686 in02
= _mm_add_epi16(in02
, kOne
);
687 in03
= _mm_add_epi16(in03
, kOne
);
688 in04
= _mm_add_epi16(in04
, kOne
);
689 in05
= _mm_add_epi16(in05
, kOne
);
690 in06
= _mm_add_epi16(in06
, kOne
);
691 in07
= _mm_add_epi16(in07
, kOne
);
692 in08
= _mm_add_epi16(in08
, kOne
);
693 in09
= _mm_add_epi16(in09
, kOne
);
694 in10
= _mm_add_epi16(in10
, kOne
);
695 in11
= _mm_add_epi16(in11
, kOne
);
696 in12
= _mm_add_epi16(in12
, kOne
);
697 in13
= _mm_add_epi16(in13
, kOne
);
698 in14
= _mm_add_epi16(in14
, kOne
);
699 in15
= _mm_add_epi16(in15
, kOne
);
700 in00
= _mm_srai_epi16(in00
, 2);
701 in01
= _mm_srai_epi16(in01
, 2);
702 in02
= _mm_srai_epi16(in02
, 2);
703 in03
= _mm_srai_epi16(in03
, 2);
704 in04
= _mm_srai_epi16(in04
, 2);
705 in05
= _mm_srai_epi16(in05
, 2);
706 in06
= _mm_srai_epi16(in06
, 2);
707 in07
= _mm_srai_epi16(in07
, 2);
708 in08
= _mm_srai_epi16(in08
, 2);
709 in09
= _mm_srai_epi16(in09
, 2);
710 in10
= _mm_srai_epi16(in10
, 2);
711 in11
= _mm_srai_epi16(in11
, 2);
712 in12
= _mm_srai_epi16(in12
, 2);
713 in13
= _mm_srai_epi16(in13
, 2);
714 in14
= _mm_srai_epi16(in14
, 2);
715 in15
= _mm_srai_epi16(in15
, 2);
718 // Calculate input for the first 8 results.
720 input0
= ADD_EPI16(in00
, in15
);
721 input1
= ADD_EPI16(in01
, in14
);
722 input2
= ADD_EPI16(in02
, in13
);
723 input3
= ADD_EPI16(in03
, in12
);
724 input4
= ADD_EPI16(in04
, in11
);
725 input5
= ADD_EPI16(in05
, in10
);
726 input6
= ADD_EPI16(in06
, in09
);
727 input7
= ADD_EPI16(in07
, in08
);
728 #if DCT_HIGH_BIT_DEPTH
729 overflow
= check_epi16_overflow_x8(&input0
, &input1
, &input2
, &input3
,
730 &input4
, &input5
, &input6
, &input7
);
732 vp9_highbd_fdct16x16_c(input
, output
, stride
);
735 #endif // DCT_HIGH_BIT_DEPTH
737 // Calculate input for the next 8 results.
739 step1_0
= SUB_EPI16(in07
, in08
);
740 step1_1
= SUB_EPI16(in06
, in09
);
741 step1_2
= SUB_EPI16(in05
, in10
);
742 step1_3
= SUB_EPI16(in04
, in11
);
743 step1_4
= SUB_EPI16(in03
, in12
);
744 step1_5
= SUB_EPI16(in02
, in13
);
745 step1_6
= SUB_EPI16(in01
, in14
);
746 step1_7
= SUB_EPI16(in00
, in15
);
747 #if DCT_HIGH_BIT_DEPTH
748 overflow
= check_epi16_overflow_x8(&step1_0
, &step1_1
,
753 vp9_highbd_fdct16x16_c(input
, output
, stride
);
756 #endif // DCT_HIGH_BIT_DEPTH
758 // Work on the first eight values; fdct8(input, even_results);
761 const __m128i q0
= ADD_EPI16(input0
, input7
);
762 const __m128i q1
= ADD_EPI16(input1
, input6
);
763 const __m128i q2
= ADD_EPI16(input2
, input5
);
764 const __m128i q3
= ADD_EPI16(input3
, input4
);
765 const __m128i q4
= SUB_EPI16(input3
, input4
);
766 const __m128i q5
= SUB_EPI16(input2
, input5
);
767 const __m128i q6
= SUB_EPI16(input1
, input6
);
768 const __m128i q7
= SUB_EPI16(input0
, input7
);
769 #if DCT_HIGH_BIT_DEPTH
770 overflow
= check_epi16_overflow_x8(&q0
, &q1
, &q2
, &q3
,
773 vp9_highbd_fdct16x16_c(input
, output
, stride
);
776 #endif // DCT_HIGH_BIT_DEPTH
777 // Work on first four results
780 const __m128i r0
= ADD_EPI16(q0
, q3
);
781 const __m128i r1
= ADD_EPI16(q1
, q2
);
782 const __m128i r2
= SUB_EPI16(q1
, q2
);
783 const __m128i r3
= SUB_EPI16(q0
, q3
);
784 #if DCT_HIGH_BIT_DEPTH
785 overflow
= check_epi16_overflow_x4(&r0
, &r1
, &r2
, &r3
);
787 vp9_highbd_fdct16x16_c(input
, output
, stride
);
790 #endif // DCT_HIGH_BIT_DEPTH
791 // Interleave to do the multiply by constants which gets us
794 const __m128i t0
= _mm_unpacklo_epi16(r0
, r1
);
795 const __m128i t1
= _mm_unpackhi_epi16(r0
, r1
);
796 const __m128i t2
= _mm_unpacklo_epi16(r2
, r3
);
797 const __m128i t3
= _mm_unpackhi_epi16(r2
, r3
);
798 res00
= mult_round_shift(&t0
, &t1
, &k__cospi_p16_p16
,
799 &k__DCT_CONST_ROUNDING
, DCT_CONST_BITS
);
800 res08
= mult_round_shift(&t0
, &t1
, &k__cospi_p16_m16
,
801 &k__DCT_CONST_ROUNDING
, DCT_CONST_BITS
);
802 res04
= mult_round_shift(&t2
, &t3
, &k__cospi_p24_p08
,
803 &k__DCT_CONST_ROUNDING
, DCT_CONST_BITS
);
804 res12
= mult_round_shift(&t2
, &t3
, &k__cospi_m08_p24
,
805 &k__DCT_CONST_ROUNDING
, DCT_CONST_BITS
);
806 #if DCT_HIGH_BIT_DEPTH
807 overflow
= check_epi16_overflow_x4(&res00
, &res08
, &res04
, &res12
);
809 vp9_highbd_fdct16x16_c(input
, output
, stride
);
812 #endif // DCT_HIGH_BIT_DEPTH
815 // Work on next four results
817 // Interleave to do the multiply by constants which gets us
819 const __m128i d0
= _mm_unpacklo_epi16(q6
, q5
);
820 const __m128i d1
= _mm_unpackhi_epi16(q6
, q5
);
821 const __m128i r0
= mult_round_shift(&d0
, &d1
, &k__cospi_p16_m16
,
822 &k__DCT_CONST_ROUNDING
,
824 const __m128i r1
= mult_round_shift(&d0
, &d1
, &k__cospi_p16_p16
,
825 &k__DCT_CONST_ROUNDING
,
827 #if DCT_HIGH_BIT_DEPTH
828 overflow
= check_epi16_overflow_x2(&r0
, &r1
);
830 vp9_highbd_fdct16x16_c(input
, output
, stride
);
833 #endif // DCT_HIGH_BIT_DEPTH
836 const __m128i x0
= ADD_EPI16(q4
, r0
);
837 const __m128i x1
= SUB_EPI16(q4
, r0
);
838 const __m128i x2
= SUB_EPI16(q7
, r1
);
839 const __m128i x3
= ADD_EPI16(q7
, r1
);
840 #if DCT_HIGH_BIT_DEPTH
841 overflow
= check_epi16_overflow_x4(&x0
, &x1
, &x2
, &x3
);
843 vp9_highbd_fdct16x16_c(input
, output
, stride
);
846 #endif // DCT_HIGH_BIT_DEPTH
847 // Interleave to do the multiply by constants which gets us
850 const __m128i t0
= _mm_unpacklo_epi16(x0
, x3
);
851 const __m128i t1
= _mm_unpackhi_epi16(x0
, x3
);
852 const __m128i t2
= _mm_unpacklo_epi16(x1
, x2
);
853 const __m128i t3
= _mm_unpackhi_epi16(x1
, x2
);
854 res02
= mult_round_shift(&t0
, &t1
, &k__cospi_p28_p04
,
855 &k__DCT_CONST_ROUNDING
, DCT_CONST_BITS
);
856 res14
= mult_round_shift(&t0
, &t1
, &k__cospi_m04_p28
,
857 &k__DCT_CONST_ROUNDING
, DCT_CONST_BITS
);
858 res10
= mult_round_shift(&t2
, &t3
, &k__cospi_p12_p20
,
859 &k__DCT_CONST_ROUNDING
, DCT_CONST_BITS
);
860 res06
= mult_round_shift(&t2
, &t3
, &k__cospi_m20_p12
,
861 &k__DCT_CONST_ROUNDING
, DCT_CONST_BITS
);
862 #if DCT_HIGH_BIT_DEPTH
863 overflow
= check_epi16_overflow_x4(&res02
, &res14
,
866 vp9_highbd_fdct16x16_c(input
, output
, stride
);
869 #endif // DCT_HIGH_BIT_DEPTH
874 // Work on the next eight values; step1 -> odd_results
878 const __m128i t0
= _mm_unpacklo_epi16(step1_5
, step1_2
);
879 const __m128i t1
= _mm_unpackhi_epi16(step1_5
, step1_2
);
880 const __m128i t2
= _mm_unpacklo_epi16(step1_4
, step1_3
);
881 const __m128i t3
= _mm_unpackhi_epi16(step1_4
, step1_3
);
882 step2_2
= mult_round_shift(&t0
, &t1
, &k__cospi_p16_m16
,
883 &k__DCT_CONST_ROUNDING
, DCT_CONST_BITS
);
884 step2_3
= mult_round_shift(&t2
, &t3
, &k__cospi_p16_m16
,
885 &k__DCT_CONST_ROUNDING
, DCT_CONST_BITS
);
886 step2_5
= mult_round_shift(&t0
, &t1
, &k__cospi_p16_p16
,
887 &k__DCT_CONST_ROUNDING
, DCT_CONST_BITS
);
888 step2_4
= mult_round_shift(&t2
, &t3
, &k__cospi_p16_p16
,
889 &k__DCT_CONST_ROUNDING
, DCT_CONST_BITS
);
890 #if DCT_HIGH_BIT_DEPTH
891 overflow
= check_epi16_overflow_x4(&step2_2
, &step2_3
, &step2_5
,
894 vp9_highbd_fdct16x16_c(input
, output
, stride
);
897 #endif // DCT_HIGH_BIT_DEPTH
901 step3_0
= ADD_EPI16(step1_0
, step2_3
);
902 step3_1
= ADD_EPI16(step1_1
, step2_2
);
903 step3_2
= SUB_EPI16(step1_1
, step2_2
);
904 step3_3
= SUB_EPI16(step1_0
, step2_3
);
905 step3_4
= SUB_EPI16(step1_7
, step2_4
);
906 step3_5
= SUB_EPI16(step1_6
, step2_5
);
907 step3_6
= ADD_EPI16(step1_6
, step2_5
);
908 step3_7
= ADD_EPI16(step1_7
, step2_4
);
909 #if DCT_HIGH_BIT_DEPTH
910 overflow
= check_epi16_overflow_x8(&step3_0
, &step3_1
,
915 vp9_highbd_fdct16x16_c(input
, output
, stride
);
918 #endif // DCT_HIGH_BIT_DEPTH
922 const __m128i t0
= _mm_unpacklo_epi16(step3_1
, step3_6
);
923 const __m128i t1
= _mm_unpackhi_epi16(step3_1
, step3_6
);
924 const __m128i t2
= _mm_unpacklo_epi16(step3_2
, step3_5
);
925 const __m128i t3
= _mm_unpackhi_epi16(step3_2
, step3_5
);
926 step2_1
= mult_round_shift(&t0
, &t1
, &k__cospi_m08_p24
,
927 &k__DCT_CONST_ROUNDING
, DCT_CONST_BITS
);
928 step2_2
= mult_round_shift(&t2
, &t3
, &k__cospi_p24_p08
,
929 &k__DCT_CONST_ROUNDING
, DCT_CONST_BITS
);
930 step2_6
= mult_round_shift(&t0
, &t1
, &k__cospi_p24_p08
,
931 &k__DCT_CONST_ROUNDING
, DCT_CONST_BITS
);
932 step2_5
= mult_round_shift(&t2
, &t3
, &k__cospi_p08_m24
,
933 &k__DCT_CONST_ROUNDING
, DCT_CONST_BITS
);
934 #if DCT_HIGH_BIT_DEPTH
935 overflow
= check_epi16_overflow_x4(&step2_1
, &step2_2
, &step2_6
,
938 vp9_highbd_fdct16x16_c(input
, output
, stride
);
941 #endif // DCT_HIGH_BIT_DEPTH
945 step1_0
= ADD_EPI16(step3_0
, step2_1
);
946 step1_1
= SUB_EPI16(step3_0
, step2_1
);
947 step1_2
= ADD_EPI16(step3_3
, step2_2
);
948 step1_3
= SUB_EPI16(step3_3
, step2_2
);
949 step1_4
= SUB_EPI16(step3_4
, step2_5
);
950 step1_5
= ADD_EPI16(step3_4
, step2_5
);
951 step1_6
= SUB_EPI16(step3_7
, step2_6
);
952 step1_7
= ADD_EPI16(step3_7
, step2_6
);
953 #if DCT_HIGH_BIT_DEPTH
954 overflow
= check_epi16_overflow_x8(&step1_0
, &step1_1
,
959 vp9_highbd_fdct16x16_c(input
, output
, stride
);
962 #endif // DCT_HIGH_BIT_DEPTH
966 const __m128i t0
= _mm_unpacklo_epi16(step1_0
, step1_7
);
967 const __m128i t1
= _mm_unpackhi_epi16(step1_0
, step1_7
);
968 const __m128i t2
= _mm_unpacklo_epi16(step1_1
, step1_6
);
969 const __m128i t3
= _mm_unpackhi_epi16(step1_1
, step1_6
);
970 res01
= mult_round_shift(&t0
, &t1
, &k__cospi_p30_p02
,
971 &k__DCT_CONST_ROUNDING
, DCT_CONST_BITS
);
972 res09
= mult_round_shift(&t2
, &t3
, &k__cospi_p14_p18
,
973 &k__DCT_CONST_ROUNDING
, DCT_CONST_BITS
);
974 res15
= mult_round_shift(&t0
, &t1
, &k__cospi_m02_p30
,
975 &k__DCT_CONST_ROUNDING
, DCT_CONST_BITS
);
976 res07
= mult_round_shift(&t2
, &t3
, &k__cospi_m18_p14
,
977 &k__DCT_CONST_ROUNDING
, DCT_CONST_BITS
);
978 #if DCT_HIGH_BIT_DEPTH
979 overflow
= check_epi16_overflow_x4(&res01
, &res09
, &res15
, &res07
);
981 vp9_highbd_fdct16x16_c(input
, output
, stride
);
984 #endif // DCT_HIGH_BIT_DEPTH
987 const __m128i t0
= _mm_unpacklo_epi16(step1_2
, step1_5
);
988 const __m128i t1
= _mm_unpackhi_epi16(step1_2
, step1_5
);
989 const __m128i t2
= _mm_unpacklo_epi16(step1_3
, step1_4
);
990 const __m128i t3
= _mm_unpackhi_epi16(step1_3
, step1_4
);
991 res05
= mult_round_shift(&t0
, &t1
, &k__cospi_p22_p10
,
992 &k__DCT_CONST_ROUNDING
, DCT_CONST_BITS
);
993 res13
= mult_round_shift(&t2
, &t3
, &k__cospi_p06_p26
,
994 &k__DCT_CONST_ROUNDING
, DCT_CONST_BITS
);
995 res11
= mult_round_shift(&t0
, &t1
, &k__cospi_m10_p22
,
996 &k__DCT_CONST_ROUNDING
, DCT_CONST_BITS
);
997 res03
= mult_round_shift(&t2
, &t3
, &k__cospi_m26_p06
,
998 &k__DCT_CONST_ROUNDING
, DCT_CONST_BITS
);
999 #if DCT_HIGH_BIT_DEPTH
1000 overflow
= check_epi16_overflow_x4(&res05
, &res13
, &res11
, &res03
);
1002 vp9_highbd_fdct16x16_c(input
, output
, stride
);
1005 #endif // DCT_HIGH_BIT_DEPTH
1008 // Transpose the results, do it as two 8x8 transposes.
1009 transpose_and_output8x8(&res00
, &res01
, &res02
, &res03
,
1010 &res04
, &res05
, &res06
, &res07
,
1012 transpose_and_output8x8(&res08
, &res09
, &res10
, &res11
,
1013 &res12
, &res13
, &res14
, &res15
,
1014 pass
, out0
+ 8, out1
+ 8);
1021 // Setup in/out for next pass.