3 * Copyright (C) 2000-2003 Michel Lespinasse <walken@zoy.org>
4 * Copyright (C) 1999-2000 Aaron Holtzman <aholtzma@ess.engr.uvic.ca>
6 * This file is part of mpeg2dec, a free MPEG-2 video stream decoder.
7 * See http://libmpeg2.sourceforge.net/ for updates.
9 * mpeg2dec is free software; you can redistribute it and/or modify
10 * it under the terms of the GNU General Public License as published by
11 * the Free Software Foundation; either version 2 of the License, or
12 * (at your option) any later version.
14 * mpeg2dec is distributed in the hope that it will be useful,
15 * but WITHOUT ANY WARRANTY; without even the implied warranty of
16 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17 * GNU General Public License for more details.
19 * You should have received a copy of the GNU General Public License
20 * along with this program; if not, write to the Free Software
21 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
23 * Modified for use with MPlayer, see libmpeg-0.4.1.diff for the exact changes.
24 * detailed changelog at http://svn.mplayerhq.hu/mplayer/trunk/
30 #if defined(ARCH_X86) || defined(ARCH_X86_64)
35 #include "attributes.h"
36 #include "mpeg2_internal.h"
42 #define round(bias) ((int)(((bias)+0.5) * (1<<ROW_SHIFT)))
43 #define rounder(bias) {round (bias), round (bias)}
47 /* C row IDCT - its just here to document the MMXEXT and MMX versions */
48 static inline void idct_row (int16_t * row
, int offset
,
49 int16_t * table
, int32_t * rounder
)
51 int C1
, C2
, C3
, C4
, C5
, C6
, C7
;
52 int a0
, a1
, a2
, a3
, b0
, b1
, b2
, b3
;
64 a0
= C4
*row
[0] + C2
*row
[2] + C4
*row
[4] + C6
*row
[6] + *rounder
;
65 a1
= C4
*row
[0] + C6
*row
[2] - C4
*row
[4] - C2
*row
[6] + *rounder
;
66 a2
= C4
*row
[0] - C6
*row
[2] - C4
*row
[4] + C2
*row
[6] + *rounder
;
67 a3
= C4
*row
[0] - C2
*row
[2] + C4
*row
[4] - C6
*row
[6] + *rounder
;
69 b0
= C1
*row
[1] + C3
*row
[3] + C5
*row
[5] + C7
*row
[7];
70 b1
= C3
*row
[1] - C7
*row
[3] - C1
*row
[5] - C5
*row
[7];
71 b2
= C5
*row
[1] - C1
*row
[3] + C7
*row
[5] + C3
*row
[7];
72 b3
= C7
*row
[1] - C5
*row
[3] + C3
*row
[5] - C1
*row
[7];
74 row
[0] = (a0
+ b0
) >> ROW_SHIFT
;
75 row
[1] = (a1
+ b1
) >> ROW_SHIFT
;
76 row
[2] = (a2
+ b2
) >> ROW_SHIFT
;
77 row
[3] = (a3
+ b3
) >> ROW_SHIFT
;
78 row
[4] = (a3
- b3
) >> ROW_SHIFT
;
79 row
[5] = (a2
- b2
) >> ROW_SHIFT
;
80 row
[6] = (a1
- b1
) >> ROW_SHIFT
;
81 row
[7] = (a0
- b0
) >> ROW_SHIFT
;
88 #define mmxext_table(c1,c2,c3,c4,c5,c6,c7) { c4, c2, -c4, -c2, \
97 static inline void mmxext_row_head (int16_t * const row
, const int offset
,
98 const int16_t * const table
)
100 movq_m2r (*(row
+offset
), mm2
); /* mm2 = x6 x4 x2 x0 */
102 movq_m2r (*(row
+offset
+4), mm5
); /* mm5 = x7 x5 x3 x1 */
103 movq_r2r (mm2
, mm0
); /* mm0 = x6 x4 x2 x0 */
105 movq_m2r (*table
, mm3
); /* mm3 = -C2 -C4 C2 C4 */
106 movq_r2r (mm5
, mm6
); /* mm6 = x7 x5 x3 x1 */
108 movq_m2r (*(table
+4), mm4
); /* mm4 = C6 C4 C6 C4 */
109 pmaddwd_r2r (mm0
, mm3
); /* mm3 = -C4*x4-C2*x6 C4*x0+C2*x2 */
111 pshufw_r2r (mm2
, mm2
, 0x4e); /* mm2 = x2 x0 x6 x4 */
114 static inline void mmxext_row (const int16_t * const table
,
115 const int32_t * const rounder
)
117 movq_m2r (*(table
+8), mm1
); /* mm1 = -C5 -C1 C3 C1 */
118 pmaddwd_r2r (mm2
, mm4
); /* mm4 = C4*x0+C6*x2 C4*x4+C6*x6 */
120 pmaddwd_m2r (*(table
+16), mm0
); /* mm0 = C4*x4-C6*x6 C4*x0-C6*x2 */
121 pshufw_r2r (mm6
, mm6
, 0x4e); /* mm6 = x3 x1 x7 x5 */
123 movq_m2r (*(table
+12), mm7
); /* mm7 = -C7 C3 C7 C5 */
124 pmaddwd_r2r (mm5
, mm1
); /* mm1 = -C1*x5-C5*x7 C1*x1+C3*x3 */
126 paddd_m2r (*rounder
, mm3
); /* mm3 += rounder */
127 pmaddwd_r2r (mm6
, mm7
); /* mm7 = C3*x1-C7*x3 C5*x5+C7*x7 */
129 pmaddwd_m2r (*(table
+20), mm2
); /* mm2 = C4*x0-C2*x2 -C4*x4+C2*x6 */
130 paddd_r2r (mm4
, mm3
); /* mm3 = a1 a0 + rounder */
132 pmaddwd_m2r (*(table
+24), mm5
); /* mm5 = C3*x5-C1*x7 C5*x1-C1*x3 */
133 movq_r2r (mm3
, mm4
); /* mm4 = a1 a0 + rounder */
135 pmaddwd_m2r (*(table
+28), mm6
); /* mm6 = C7*x1-C5*x3 C7*x5+C3*x7 */
136 paddd_r2r (mm7
, mm1
); /* mm1 = b1 b0 */
138 paddd_m2r (*rounder
, mm0
); /* mm0 += rounder */
139 psubd_r2r (mm1
, mm3
); /* mm3 = a1-b1 a0-b0 + rounder */
141 psrad_i2r (ROW_SHIFT
, mm3
); /* mm3 = y6 y7 */
142 paddd_r2r (mm4
, mm1
); /* mm1 = a1+b1 a0+b0 + rounder */
144 paddd_r2r (mm2
, mm0
); /* mm0 = a3 a2 + rounder */
145 psrad_i2r (ROW_SHIFT
, mm1
); /* mm1 = y1 y0 */
147 paddd_r2r (mm6
, mm5
); /* mm5 = b3 b2 */
148 movq_r2r (mm0
, mm4
); /* mm4 = a3 a2 + rounder */
150 paddd_r2r (mm5
, mm0
); /* mm0 = a3+b3 a2+b2 + rounder */
151 psubd_r2r (mm5
, mm4
); /* mm4 = a3-b3 a2-b2 + rounder */
154 static inline void mmxext_row_tail (int16_t * const row
, const int store
)
156 psrad_i2r (ROW_SHIFT
, mm0
); /* mm0 = y3 y2 */
158 psrad_i2r (ROW_SHIFT
, mm4
); /* mm4 = y4 y5 */
160 packssdw_r2r (mm0
, mm1
); /* mm1 = y3 y2 y1 y0 */
162 packssdw_r2r (mm3
, mm4
); /* mm4 = y6 y7 y4 y5 */
164 movq_r2m (mm1
, *(row
+store
)); /* save y3 y2 y1 y0 */
165 pshufw_r2r (mm4
, mm4
, 0xb1); /* mm4 = y7 y6 y5 y4 */
169 movq_r2m (mm4
, *(row
+store
+4)); /* save y7 y6 y5 y4 */
172 static inline void mmxext_row_mid (int16_t * const row
, const int store
,
174 const int16_t * const table
)
176 movq_m2r (*(row
+offset
), mm2
); /* mm2 = x6 x4 x2 x0 */
177 psrad_i2r (ROW_SHIFT
, mm0
); /* mm0 = y3 y2 */
179 movq_m2r (*(row
+offset
+4), mm5
); /* mm5 = x7 x5 x3 x1 */
180 psrad_i2r (ROW_SHIFT
, mm4
); /* mm4 = y4 y5 */
182 packssdw_r2r (mm0
, mm1
); /* mm1 = y3 y2 y1 y0 */
183 movq_r2r (mm5
, mm6
); /* mm6 = x7 x5 x3 x1 */
185 packssdw_r2r (mm3
, mm4
); /* mm4 = y6 y7 y4 y5 */
186 movq_r2r (mm2
, mm0
); /* mm0 = x6 x4 x2 x0 */
188 movq_r2m (mm1
, *(row
+store
)); /* save y3 y2 y1 y0 */
189 pshufw_r2r (mm4
, mm4
, 0xb1); /* mm4 = y7 y6 y5 y4 */
191 movq_m2r (*table
, mm3
); /* mm3 = -C2 -C4 C2 C4 */
192 movq_r2m (mm4
, *(row
+store
+4)); /* save y7 y6 y5 y4 */
194 pmaddwd_r2r (mm0
, mm3
); /* mm3 = -C4*x4-C2*x6 C4*x0+C2*x2 */
196 movq_m2r (*(table
+4), mm4
); /* mm4 = C6 C4 C6 C4 */
197 pshufw_r2r (mm2
, mm2
, 0x4e); /* mm2 = x2 x0 x6 x4 */
203 #define mmx_table(c1,c2,c3,c4,c5,c6,c7) { c4, c2, c4, c6, \
212 static inline void mmx_row_head (int16_t * const row
, const int offset
,
213 const int16_t * const table
)
215 movq_m2r (*(row
+offset
), mm2
); /* mm2 = x6 x4 x2 x0 */
217 movq_m2r (*(row
+offset
+4), mm5
); /* mm5 = x7 x5 x3 x1 */
218 movq_r2r (mm2
, mm0
); /* mm0 = x6 x4 x2 x0 */
220 movq_m2r (*table
, mm3
); /* mm3 = C6 C4 C2 C4 */
221 movq_r2r (mm5
, mm6
); /* mm6 = x7 x5 x3 x1 */
223 punpckldq_r2r (mm0
, mm0
); /* mm0 = x2 x0 x2 x0 */
225 movq_m2r (*(table
+4), mm4
); /* mm4 = -C2 -C4 C6 C4 */
226 pmaddwd_r2r (mm0
, mm3
); /* mm3 = C4*x0+C6*x2 C4*x0+C2*x2 */
228 movq_m2r (*(table
+8), mm1
); /* mm1 = -C7 C3 C3 C1 */
229 punpckhdq_r2r (mm2
, mm2
); /* mm2 = x6 x4 x6 x4 */
232 static inline void mmx_row (const int16_t * const table
,
233 const int32_t * const rounder
)
235 pmaddwd_r2r (mm2
, mm4
); /* mm4 = -C4*x4-C2*x6 C4*x4+C6*x6 */
236 punpckldq_r2r (mm5
, mm5
); /* mm5 = x3 x1 x3 x1 */
238 pmaddwd_m2r (*(table
+16), mm0
); /* mm0 = C4*x0-C2*x2 C4*x0-C6*x2 */
239 punpckhdq_r2r (mm6
, mm6
); /* mm6 = x7 x5 x7 x5 */
241 movq_m2r (*(table
+12), mm7
); /* mm7 = -C5 -C1 C7 C5 */
242 pmaddwd_r2r (mm5
, mm1
); /* mm1 = C3*x1-C7*x3 C1*x1+C3*x3 */
244 paddd_m2r (*rounder
, mm3
); /* mm3 += rounder */
245 pmaddwd_r2r (mm6
, mm7
); /* mm7 = -C1*x5-C5*x7 C5*x5+C7*x7 */
247 pmaddwd_m2r (*(table
+20), mm2
); /* mm2 = C4*x4-C6*x6 -C4*x4+C2*x6 */
248 paddd_r2r (mm4
, mm3
); /* mm3 = a1 a0 + rounder */
250 pmaddwd_m2r (*(table
+24), mm5
); /* mm5 = C7*x1-C5*x3 C5*x1-C1*x3 */
251 movq_r2r (mm3
, mm4
); /* mm4 = a1 a0 + rounder */
253 pmaddwd_m2r (*(table
+28), mm6
); /* mm6 = C3*x5-C1*x7 C7*x5+C3*x7 */
254 paddd_r2r (mm7
, mm1
); /* mm1 = b1 b0 */
256 paddd_m2r (*rounder
, mm0
); /* mm0 += rounder */
257 psubd_r2r (mm1
, mm3
); /* mm3 = a1-b1 a0-b0 + rounder */
259 psrad_i2r (ROW_SHIFT
, mm3
); /* mm3 = y6 y7 */
260 paddd_r2r (mm4
, mm1
); /* mm1 = a1+b1 a0+b0 + rounder */
262 paddd_r2r (mm2
, mm0
); /* mm0 = a3 a2 + rounder */
263 psrad_i2r (ROW_SHIFT
, mm1
); /* mm1 = y1 y0 */
265 paddd_r2r (mm6
, mm5
); /* mm5 = b3 b2 */
266 movq_r2r (mm0
, mm7
); /* mm7 = a3 a2 + rounder */
268 paddd_r2r (mm5
, mm0
); /* mm0 = a3+b3 a2+b2 + rounder */
269 psubd_r2r (mm5
, mm7
); /* mm7 = a3-b3 a2-b2 + rounder */
272 static inline void mmx_row_tail (int16_t * const row
, const int store
)
274 psrad_i2r (ROW_SHIFT
, mm0
); /* mm0 = y3 y2 */
276 psrad_i2r (ROW_SHIFT
, mm7
); /* mm7 = y4 y5 */
278 packssdw_r2r (mm0
, mm1
); /* mm1 = y3 y2 y1 y0 */
280 packssdw_r2r (mm3
, mm7
); /* mm7 = y6 y7 y4 y5 */
282 movq_r2m (mm1
, *(row
+store
)); /* save y3 y2 y1 y0 */
283 movq_r2r (mm7
, mm4
); /* mm4 = y6 y7 y4 y5 */
285 pslld_i2r (16, mm7
); /* mm7 = y7 0 y5 0 */
287 psrld_i2r (16, mm4
); /* mm4 = 0 y6 0 y4 */
289 por_r2r (mm4
, mm7
); /* mm7 = y7 y6 y5 y4 */
293 movq_r2m (mm7
, *(row
+store
+4)); /* save y7 y6 y5 y4 */
296 static inline void mmx_row_mid (int16_t * const row
, const int store
,
297 const int offset
, const int16_t * const table
)
299 movq_m2r (*(row
+offset
), mm2
); /* mm2 = x6 x4 x2 x0 */
300 psrad_i2r (ROW_SHIFT
, mm0
); /* mm0 = y3 y2 */
302 movq_m2r (*(row
+offset
+4), mm5
); /* mm5 = x7 x5 x3 x1 */
303 psrad_i2r (ROW_SHIFT
, mm7
); /* mm7 = y4 y5 */
305 packssdw_r2r (mm0
, mm1
); /* mm1 = y3 y2 y1 y0 */
306 movq_r2r (mm5
, mm6
); /* mm6 = x7 x5 x3 x1 */
308 packssdw_r2r (mm3
, mm7
); /* mm7 = y6 y7 y4 y5 */
309 movq_r2r (mm2
, mm0
); /* mm0 = x6 x4 x2 x0 */
311 movq_r2m (mm1
, *(row
+store
)); /* save y3 y2 y1 y0 */
312 movq_r2r (mm7
, mm1
); /* mm1 = y6 y7 y4 y5 */
314 punpckldq_r2r (mm0
, mm0
); /* mm0 = x2 x0 x2 x0 */
315 psrld_i2r (16, mm7
); /* mm7 = 0 y6 0 y4 */
317 movq_m2r (*table
, mm3
); /* mm3 = C6 C4 C2 C4 */
318 pslld_i2r (16, mm1
); /* mm1 = y7 0 y5 0 */
320 movq_m2r (*(table
+4), mm4
); /* mm4 = -C2 -C4 C6 C4 */
321 por_r2r (mm1
, mm7
); /* mm7 = y7 y6 y5 y4 */
323 movq_m2r (*(table
+8), mm1
); /* mm1 = -C7 C3 C3 C1 */
324 punpckhdq_r2r (mm2
, mm2
); /* mm2 = x6 x4 x6 x4 */
326 movq_r2m (mm7
, *(row
+store
+4)); /* save y7 y6 y5 y4 */
327 pmaddwd_r2r (mm0
, mm3
); /* mm3 = C4*x0+C6*x2 C4*x0+C2*x2 */
332 /* C column IDCT - its just here to document the MMXEXT and MMX versions */
333 static inline void idct_col (int16_t * col
, int offset
)
335 /* multiplication - as implemented on mmx */
336 #define F(c,x) (((c) * (x)) >> 16)
338 /* saturation - it helps us handle torture test cases */
339 #define S(x) (((x)>32767) ? 32767 : ((x)<-32768) ? -32768 : (x))
341 int16_t x0
, x1
, x2
, x3
, x4
, x5
, x6
, x7
;
342 int16_t y0
, y1
, y2
, y3
, y4
, y5
, y6
, y7
;
343 int16_t a0
, a1
, a2
, a3
, b0
, b1
, b2
, b3
;
344 int16_t u04
, v04
, u26
, v26
, u17
, v17
, u35
, v35
, u12
, v12
;
359 u26
= S (F (T2
, x6
) + x2
);
360 v26
= S (F (T2
, x2
) - x6
);
367 u17
= S (F (T1
, x7
) + x1
);
368 v17
= S (F (T1
, x1
) - x7
);
369 u35
= S (F (T3
, x5
) + x3
);
370 v35
= S (F (T3
, x3
) - x5
);
376 u12
= S (2 * F (C4
, u12
));
377 v12
= S (2 * F (C4
, v12
));
381 y0
= S (a0
+ b0
) >> COL_SHIFT
;
382 y1
= S (a1
+ b1
) >> COL_SHIFT
;
383 y2
= S (a2
+ b2
) >> COL_SHIFT
;
384 y3
= S (a3
+ b3
) >> COL_SHIFT
;
386 y4
= S (a3
- b3
) >> COL_SHIFT
;
387 y5
= S (a2
- b2
) >> COL_SHIFT
;
388 y6
= S (a1
- b1
) >> COL_SHIFT
;
389 y7
= S (a0
- b0
) >> COL_SHIFT
;
403 /* MMX column IDCT */
404 static inline void idct_col (int16_t * const col
, const int offset
)
411 static const short _T1
[] ATTR_ALIGN(8) = {T1
,T1
,T1
,T1
};
412 static const short _T2
[] ATTR_ALIGN(8) = {T2
,T2
,T2
,T2
};
413 static const short _T3
[] ATTR_ALIGN(8) = {T3
,T3
,T3
,T3
};
414 static const short _C4
[] ATTR_ALIGN(8) = {C4
,C4
,C4
,C4
};
416 /* column code adapted from peter gubanov */
417 /* http://www.elecard.com/peter/idct.shtml */
419 movq_m2r (*_T1
, mm0
); /* mm0 = T1 */
421 movq_m2r (*(col
+offset
+1*8), mm1
); /* mm1 = x1 */
422 movq_r2r (mm0
, mm2
); /* mm2 = T1 */
424 movq_m2r (*(col
+offset
+7*8), mm4
); /* mm4 = x7 */
425 pmulhw_r2r (mm1
, mm0
); /* mm0 = T1*x1 */
427 movq_m2r (*_T3
, mm5
); /* mm5 = T3 */
428 pmulhw_r2r (mm4
, mm2
); /* mm2 = T1*x7 */
430 movq_m2r (*(col
+offset
+5*8), mm6
); /* mm6 = x5 */
431 movq_r2r (mm5
, mm7
); /* mm7 = T3-1 */
433 movq_m2r (*(col
+offset
+3*8), mm3
); /* mm3 = x3 */
434 psubsw_r2r (mm4
, mm0
); /* mm0 = v17 */
436 movq_m2r (*_T2
, mm4
); /* mm4 = T2 */
437 pmulhw_r2r (mm3
, mm5
); /* mm5 = (T3-1)*x3 */
439 paddsw_r2r (mm2
, mm1
); /* mm1 = u17 */
440 pmulhw_r2r (mm6
, mm7
); /* mm7 = (T3-1)*x5 */
444 movq_r2r (mm4
, mm2
); /* mm2 = T2 */
445 paddsw_r2r (mm3
, mm5
); /* mm5 = T3*x3 */
447 pmulhw_m2r (*(col
+offset
+2*8), mm4
);/* mm4 = T2*x2 */
448 paddsw_r2r (mm6
, mm7
); /* mm7 = T3*x5 */
450 psubsw_r2r (mm6
, mm5
); /* mm5 = v35 */
451 paddsw_r2r (mm3
, mm7
); /* mm7 = u35 */
453 movq_m2r (*(col
+offset
+6*8), mm3
); /* mm3 = x6 */
454 movq_r2r (mm0
, mm6
); /* mm6 = v17 */
456 pmulhw_r2r (mm3
, mm2
); /* mm2 = T2*x6 */
457 psubsw_r2r (mm5
, mm0
); /* mm0 = b3 */
459 psubsw_r2r (mm3
, mm4
); /* mm4 = v26 */
460 paddsw_r2r (mm6
, mm5
); /* mm5 = v12 */
462 movq_r2m (mm0
, *(col
+offset
+3*8)); /* save b3 in scratch0 */
463 movq_r2r (mm1
, mm6
); /* mm6 = u17 */
465 paddsw_m2r (*(col
+offset
+2*8), mm2
);/* mm2 = u26 */
466 paddsw_r2r (mm7
, mm6
); /* mm6 = b0 */
468 psubsw_r2r (mm7
, mm1
); /* mm1 = u12 */
469 movq_r2r (mm1
, mm7
); /* mm7 = u12 */
471 movq_m2r (*(col
+offset
+0*8), mm3
); /* mm3 = x0 */
472 paddsw_r2r (mm5
, mm1
); /* mm1 = u12+v12 */
474 movq_m2r (*_C4
, mm0
); /* mm0 = C4/2 */
475 psubsw_r2r (mm5
, mm7
); /* mm7 = u12-v12 */
477 movq_r2m (mm6
, *(col
+offset
+5*8)); /* save b0 in scratch1 */
478 pmulhw_r2r (mm0
, mm1
); /* mm1 = b1/2 */
480 movq_r2r (mm4
, mm6
); /* mm6 = v26 */
481 pmulhw_r2r (mm0
, mm7
); /* mm7 = b2/2 */
483 movq_m2r (*(col
+offset
+4*8), mm5
); /* mm5 = x4 */
484 movq_r2r (mm3
, mm0
); /* mm0 = x0 */
486 psubsw_r2r (mm5
, mm3
); /* mm3 = v04 */
487 paddsw_r2r (mm5
, mm0
); /* mm0 = u04 */
489 paddsw_r2r (mm3
, mm4
); /* mm4 = a1 */
490 movq_r2r (mm0
, mm5
); /* mm5 = u04 */
492 psubsw_r2r (mm6
, mm3
); /* mm3 = a2 */
493 paddsw_r2r (mm2
, mm5
); /* mm5 = a0 */
495 paddsw_r2r (mm1
, mm1
); /* mm1 = b1 */
496 psubsw_r2r (mm2
, mm0
); /* mm0 = a3 */
498 paddsw_r2r (mm7
, mm7
); /* mm7 = b2 */
499 movq_r2r (mm3
, mm2
); /* mm2 = a2 */
501 movq_r2r (mm4
, mm6
); /* mm6 = a1 */
502 paddsw_r2r (mm7
, mm3
); /* mm3 = a2+b2 */
504 psraw_i2r (COL_SHIFT
, mm3
); /* mm3 = y2 */
505 paddsw_r2r (mm1
, mm4
); /* mm4 = a1+b1 */
507 psraw_i2r (COL_SHIFT
, mm4
); /* mm4 = y1 */
508 psubsw_r2r (mm1
, mm6
); /* mm6 = a1-b1 */
510 movq_m2r (*(col
+offset
+5*8), mm1
); /* mm1 = b0 */
511 psubsw_r2r (mm7
, mm2
); /* mm2 = a2-b2 */
513 psraw_i2r (COL_SHIFT
, mm6
); /* mm6 = y6 */
514 movq_r2r (mm5
, mm7
); /* mm7 = a0 */
516 movq_r2m (mm4
, *(col
+offset
+1*8)); /* save y1 */
517 psraw_i2r (COL_SHIFT
, mm2
); /* mm2 = y5 */
519 movq_r2m (mm3
, *(col
+offset
+2*8)); /* save y2 */
520 paddsw_r2r (mm1
, mm5
); /* mm5 = a0+b0 */
522 movq_m2r (*(col
+offset
+3*8), mm4
); /* mm4 = b3 */
523 psubsw_r2r (mm1
, mm7
); /* mm7 = a0-b0 */
525 psraw_i2r (COL_SHIFT
, mm5
); /* mm5 = y0 */
526 movq_r2r (mm0
, mm3
); /* mm3 = a3 */
528 movq_r2m (mm2
, *(col
+offset
+5*8)); /* save y5 */
529 psubsw_r2r (mm4
, mm3
); /* mm3 = a3-b3 */
531 psraw_i2r (COL_SHIFT
, mm7
); /* mm7 = y7 */
532 paddsw_r2r (mm0
, mm4
); /* mm4 = a3+b3 */
534 movq_r2m (mm5
, *(col
+offset
+0*8)); /* save y0 */
535 psraw_i2r (COL_SHIFT
, mm3
); /* mm3 = y4 */
537 movq_r2m (mm6
, *(col
+offset
+6*8)); /* save y6 */
538 psraw_i2r (COL_SHIFT
, mm4
); /* mm4 = y3 */
540 movq_r2m (mm7
, *(col
+offset
+7*8)); /* save y7 */
542 movq_r2m (mm3
, *(col
+offset
+4*8)); /* save y4 */
544 movq_r2m (mm4
, *(col
+offset
+3*8)); /* save y3 */
548 static const int32_t rounder0
[] ATTR_ALIGN(8) =
549 rounder ((1 << (COL_SHIFT
- 1)) - 0.5);
550 static const int32_t rounder4
[] ATTR_ALIGN(8) = rounder (0);
551 static const int32_t rounder1
[] ATTR_ALIGN(8) =
552 rounder (1.25683487303); /* C1*(C1/C4+C1+C7)/2 */
553 static const int32_t rounder7
[] ATTR_ALIGN(8) =
554 rounder (-0.25); /* C1*(C7/C4+C7-C1)/2 */
555 static const int32_t rounder2
[] ATTR_ALIGN(8) =
556 rounder (0.60355339059); /* C2 * (C6+C2)/2 */
557 static const int32_t rounder6
[] ATTR_ALIGN(8) =
558 rounder (-0.25); /* C2 * (C6-C2)/2 */
559 static const int32_t rounder3
[] ATTR_ALIGN(8) =
560 rounder (0.087788325588); /* C3*(-C3/C4+C3+C5)/2 */
561 static const int32_t rounder5
[] ATTR_ALIGN(8) =
562 rounder (-0.441341716183); /* C3*(-C5/C4+C5-C3)/2 */
565 #define declare_idct(idct,table,idct_row_head,idct_row,idct_row_tail,idct_row_mid) \
566 static inline void idct (int16_t * const block) \
568 static const int16_t table04[] ATTR_ALIGN(16) = \
569 table (22725, 21407, 19266, 16384, 12873, 8867, 4520); \
570 static const int16_t table17[] ATTR_ALIGN(16) = \
571 table (31521, 29692, 26722, 22725, 17855, 12299, 6270); \
572 static const int16_t table26[] ATTR_ALIGN(16) = \
573 table (29692, 27969, 25172, 21407, 16819, 11585, 5906); \
574 static const int16_t table35[] ATTR_ALIGN(16) = \
575 table (26722, 25172, 22654, 19266, 15137, 10426, 5315); \
577 idct_row_head (block, 0*8, table04); \
578 idct_row (table04, rounder0); \
579 idct_row_mid (block, 0*8, 4*8, table04); \
580 idct_row (table04, rounder4); \
581 idct_row_mid (block, 4*8, 1*8, table17); \
582 idct_row (table17, rounder1); \
583 idct_row_mid (block, 1*8, 7*8, table17); \
584 idct_row (table17, rounder7); \
585 idct_row_mid (block, 7*8, 2*8, table26); \
586 idct_row (table26, rounder2); \
587 idct_row_mid (block, 2*8, 6*8, table26); \
588 idct_row (table26, rounder6); \
589 idct_row_mid (block, 6*8, 3*8, table35); \
590 idct_row (table35, rounder3); \
591 idct_row_mid (block, 3*8, 5*8, table35); \
592 idct_row (table35, rounder5); \
593 idct_row_tail (block, 5*8); \
595 idct_col (block, 0); \
596 idct_col (block, 4); \
600 #define COPY_MMX(offset,r0,r1,r2) \
602 movq_m2r (*(block+offset), r0); \
604 movq_m2r (*(block+offset+4), r1); \
605 movq_r2m (r2, *dest); \
606 packuswb_r2r (r1, r0); \
609 static inline void block_copy (int16_t * const block
, uint8_t * dest
,
612 movq_m2r (*(block
+0*8), mm0
);
613 movq_m2r (*(block
+0*8+4), mm1
);
614 movq_m2r (*(block
+1*8), mm2
);
615 packuswb_r2r (mm1
, mm0
);
616 movq_m2r (*(block
+1*8+4), mm3
);
617 movq_r2m (mm0
, *dest
);
618 packuswb_r2r (mm3
, mm2
);
619 COPY_MMX (2*8, mm0
, mm1
, mm2
);
620 COPY_MMX (3*8, mm2
, mm3
, mm0
);
621 COPY_MMX (4*8, mm0
, mm1
, mm2
);
622 COPY_MMX (5*8, mm2
, mm3
, mm0
);
623 COPY_MMX (6*8, mm0
, mm1
, mm2
);
624 COPY_MMX (7*8, mm2
, mm3
, mm0
);
625 movq_r2m (mm2
, *(dest
+stride
));
629 #define ADD_MMX(offset,r1,r2,r3,r4) \
631 movq_m2r (*(dest+2*stride), r1); \
632 packuswb_r2r (r4, r3); \
635 movq_r2m (r3, *dest); \
636 punpcklbw_r2r (mm0, r1); \
637 paddsw_m2r (*(block+offset), r1); \
638 punpckhbw_r2r (mm0, r2); \
639 paddsw_m2r (*(block+offset+4), r2); \
642 static inline void block_add (int16_t * const block
, uint8_t * dest
,
645 movq_m2r (*dest
, mm1
);
647 movq_m2r (*(dest
+stride
), mm3
);
649 punpcklbw_r2r (mm0
, mm1
);
651 paddsw_m2r (*(block
+0*8), mm1
);
652 punpckhbw_r2r (mm0
, mm2
);
653 paddsw_m2r (*(block
+0*8+4), mm2
);
654 punpcklbw_r2r (mm0
, mm3
);
655 paddsw_m2r (*(block
+1*8), mm3
);
656 packuswb_r2r (mm2
, mm1
);
657 punpckhbw_r2r (mm0
, mm4
);
658 movq_r2m (mm1
, *dest
);
659 paddsw_m2r (*(block
+1*8+4), mm4
);
660 ADD_MMX (2*8, mm1
, mm2
, mm3
, mm4
);
661 ADD_MMX (3*8, mm3
, mm4
, mm1
, mm2
);
662 ADD_MMX (4*8, mm1
, mm2
, mm3
, mm4
);
663 ADD_MMX (5*8, mm3
, mm4
, mm1
, mm2
);
664 ADD_MMX (6*8, mm1
, mm2
, mm3
, mm4
);
665 ADD_MMX (7*8, mm3
, mm4
, mm1
, mm2
);
666 packuswb_r2r (mm4
, mm3
);
667 movq_r2m (mm3
, *(dest
+stride
));
671 static inline void block_zero (int16_t * const block
)
674 movq_r2m (mm0
, *(block
+0*4));
675 movq_r2m (mm0
, *(block
+1*4));
676 movq_r2m (mm0
, *(block
+2*4));
677 movq_r2m (mm0
, *(block
+3*4));
678 movq_r2m (mm0
, *(block
+4*4));
679 movq_r2m (mm0
, *(block
+5*4));
680 movq_r2m (mm0
, *(block
+6*4));
681 movq_r2m (mm0
, *(block
+7*4));
682 movq_r2m (mm0
, *(block
+8*4));
683 movq_r2m (mm0
, *(block
+9*4));
684 movq_r2m (mm0
, *(block
+10*4));
685 movq_r2m (mm0
, *(block
+11*4));
686 movq_r2m (mm0
, *(block
+12*4));
687 movq_r2m (mm0
, *(block
+13*4));
688 movq_r2m (mm0
, *(block
+14*4));
689 movq_r2m (mm0
, *(block
+15*4));
698 if (cpu != CPU_MMXEXT) { \
699 punpcklwd_r2r (reg, reg); \
700 punpckldq_r2r (reg, reg); \
702 pshufw_r2r (reg, reg, 0x00); \
705 static inline void block_add_DC (int16_t * const block
, uint8_t * dest
,
706 const int stride
, const int cpu
)
708 movd_v2r ((block
[0] + 64) >> 7, mm0
);
710 movq_m2r (*dest
, mm2
);
712 psubsw_r2r (mm0
, mm1
);
713 packuswb_r2r (mm0
, mm0
);
714 paddusb_r2r (mm0
, mm2
);
715 packuswb_r2r (mm1
, mm1
);
716 movq_m2r (*(dest
+ stride
), mm3
);
717 psubusb_r2r (mm1
, mm2
);
719 paddusb_r2r (mm0
, mm3
);
720 movq_r2m (mm2
, *dest
);
721 psubusb_r2r (mm1
, mm3
);
722 movq_m2r (*(dest
+ 2*stride
), mm2
);
724 movq_r2m (mm3
, *dest
);
725 paddusb_r2r (mm0
, mm2
);
726 movq_m2r (*(dest
+ 2*stride
), mm3
);
727 psubusb_r2r (mm1
, mm2
);
729 paddusb_r2r (mm0
, mm3
);
730 movq_r2m (mm2
, *dest
);
731 psubusb_r2r (mm1
, mm3
);
732 movq_m2r (*(dest
+ 2*stride
), mm2
);
734 movq_r2m (mm3
, *dest
);
735 paddusb_r2r (mm0
, mm2
);
736 movq_m2r (*(dest
+ 2*stride
), mm3
);
737 psubusb_r2r (mm1
, mm2
);
739 paddusb_r2r (mm0
, mm3
);
740 movq_r2m (mm2
, *dest
);
741 psubusb_r2r (mm1
, mm3
);
742 movq_m2r (*(dest
+ 2*stride
), mm2
);
744 movq_r2m (mm3
, *dest
);
745 paddusb_r2r (mm0
, mm2
);
746 movq_m2r (*(dest
+ 2*stride
), mm3
);
747 psubusb_r2r (mm1
, mm2
);
749 paddusb_r2r (mm0
, mm3
);
750 movq_r2m (mm2
, *(dest
+ stride
));
751 psubusb_r2r (mm1
, mm3
);
752 movq_r2m (mm3
, *(dest
+ 2*stride
));
756 declare_idct (mmxext_idct
, mmxext_table
,
757 mmxext_row_head
, mmxext_row
, mmxext_row_tail
, mmxext_row_mid
)
759 void mpeg2_idct_copy_mmxext (int16_t * const block
, uint8_t * const dest
,
763 block_copy (block
, dest
, stride
);
767 void mpeg2_idct_add_mmxext (const int last
, int16_t * const block
,
768 uint8_t * const dest
, const int stride
)
770 if (last
!= 129 || (block
[0] & (7 << 4)) == (4 << 4)) {
772 block_add (block
, dest
, stride
);
775 block_add_DC (block
, dest
, stride
, CPU_MMXEXT
);
779 declare_idct (mmx_idct
, mmx_table
,
780 mmx_row_head
, mmx_row
, mmx_row_tail
, mmx_row_mid
)
782 void mpeg2_idct_copy_mmx (int16_t * const block
, uint8_t * const dest
,
786 block_copy (block
, dest
, stride
);
790 void mpeg2_idct_add_mmx (const int last
, int16_t * const block
,
791 uint8_t * const dest
, const int stride
)
793 if (last
!= 129 || (block
[0] & (7 << 4)) == (4 << 4)) {
795 block_add (block
, dest
, stride
);
798 block_add_DC (block
, dest
, stride
, CPU_MMX
);
802 void mpeg2_idct_mmx_init (void)
804 extern uint8_t mpeg2_scan_norm
[64];
805 extern uint8_t mpeg2_scan_alt
[64];
808 /* the mmx/mmxext idct uses a reordered input, so we patch scan tables */
810 for (i
= 0; i
< 64; i
++) {
811 j
= mpeg2_scan_norm
[i
];
812 mpeg2_scan_norm
[i
] = (j
& 0x38) | ((j
& 6) >> 1) | ((j
& 1) << 2);
813 j
= mpeg2_scan_alt
[i
];
814 mpeg2_scan_alt
[i
] = (j
& 0x38) | ((j
& 6) >> 1) | ((j
& 1) << 2);