mp3lib/dct64_k7.c

   1 /*
   2 * This code was taken from http://www.mpg123.org
   3 * See ChangeLog of mpg123-0.59s-pre.1 for detail
   4 * Applied to mplayer by Nick Kurshev <nickols_k@mail.ru>
   5 * Partial 3dnowex-DSP! optimization by Nick Kurshev
   6 *
   7 * TODO: optimize scalar 3dnow! code
   8 * Warning: Phases 7 & 8 are not tested
   9 */
  10 #define real float /* ugly - but only way */
  11
  12 #include "../mangle.h"
  13
  14 static unsigned long long int __attribute__((aligned(8))) x_plus_minus_3dnow = 0x8000000000000000ULL;
  15 static float plus_1f = 1.0;
  16
  17 void __attribute__ (( __stdcall__ )) dct64_MMX_3dnowex(real *a,real *b,real *c)
  18 {
  19   char tmp[256];
  20     __asm __volatile(
  21 "       movl %2,%%eax\n\t"
  22
  23 "       leal 128+%3,%%edx\n\t"
  24 "       movl %0,%%esi\n\t"
  25 "       movl %1,%%edi\n\t"
  26 "       movl $"MANGLE(costab_mmx)",%%ebx\n\t"
  27 "       leal %3,%%ecx\n\t"
  28
  29 /* Phase 1*/
  30 "       movq    (%%eax), %%mm0\n\t"
  31 "       movq    8(%%eax), %%mm4\n\t"
  32 "       movq    %%mm0, %%mm3\n\t"
  33 "       movq    %%mm4, %%mm7\n\t"
  34 "       pshufw  $78, 120(%%eax), %%mm1\n\t"
  35 "       pshufw  $78, 112(%%eax), %%mm5\n\t"
  36 "       pfadd   %%mm1, %%mm0\n\t"
  37 "       pfadd   %%mm5, %%mm4\n\t"
  38 "       movq    %%mm0, (%%edx)\n\t"
  39 "       movq    %%mm4, 8(%%edx)\n\t"
  40 "       pfsub   %%mm1, %%mm3\n\t"
  41 "       pfsub   %%mm5, %%mm7\n\t"
  42 "       pfmul   (%%ebx), %%mm3\n\t"
  43 "       pfmul   8(%%ebx), %%mm7\n\t"
  44 "       pswapd  %%mm3, %%mm3\n\t"
  45 "       pswapd  %%mm7, %%mm7\n\t"
  46 "       movq    %%mm3, 120(%%edx)\n\t"
  47 "       movq    %%mm7, 112(%%edx)\n\t"
  48
  49 "       movq    16(%%eax), %%mm0\n\t"
  50 "       movq    24(%%eax), %%mm4\n\t"
  51 "       movq    %%mm0, %%mm3\n\t"
  52 "       movq    %%mm4, %%mm7\n\t"
  53 "       pshufw  $78, 104(%%eax), %%mm1\n\t"
  54 "       pshufw  $78, 96(%%eax), %%mm5\n\t"
  55 "       pfadd   %%mm1, %%mm0\n\t"
  56 "       pfadd   %%mm5, %%mm4\n\t"
  57 "       movq    %%mm0, 16(%%edx)\n\t"
  58 "       movq    %%mm4, 24(%%edx)\n\t"
  59 "       pfsub   %%mm1, %%mm3\n\t"
  60 "       pfsub   %%mm5, %%mm7\n\t"
  61 "       pfmul   16(%%ebx), %%mm3\n\t"
  62 "       pfmul   24(%%ebx), %%mm7\n\t"
  63 "       pswapd  %%mm3, %%mm3\n\t"
  64 "       pswapd  %%mm7, %%mm7\n\t"
  65 "       movq    %%mm3, 104(%%edx)\n\t"
  66 "       movq    %%mm7, 96(%%edx)\n\t"
  67
  68 "       movq    32(%%eax), %%mm0\n\t"
  69 "       movq    40(%%eax), %%mm4\n\t"
  70 "       movq    %%mm0, %%mm3\n\t"
  71 "       movq    %%mm4, %%mm7\n\t"
  72 "       pshufw  $78, 88(%%eax), %%mm1\n\t"
  73 "       pshufw  $78, 80(%%eax), %%mm5\n\t"
  74 "       pfadd   %%mm1, %%mm0\n\t"
  75 "       pfadd   %%mm5, %%mm4\n\t"
  76 "       movq    %%mm0, 32(%%edx)\n\t"
  77 "       movq    %%mm4, 40(%%edx)\n\t"
  78 "       pfsub   %%mm1, %%mm3\n\t"
  79 "       pfsub   %%mm5, %%mm7\n\t"
  80 "       pfmul   32(%%ebx), %%mm3\n\t"
  81 "       pfmul   40(%%ebx), %%mm7\n\t"
  82 "       pswapd  %%mm3, %%mm3\n\t"
  83 "       pswapd  %%mm7, %%mm7\n\t"
  84 "       movq    %%mm3, 88(%%edx)\n\t"
  85 "       movq    %%mm7, 80(%%edx)\n\t"
  86
  87 "       movq    48(%%eax), %%mm0\n\t"
  88 "       movq    56(%%eax), %%mm4\n\t"
  89 "       movq    %%mm0, %%mm3\n\t"
  90 "       movq    %%mm4, %%mm7\n\t"
  91 "       pshufw  $78, 72(%%eax), %%mm1\n\t"
  92 "       pshufw  $78, 64(%%eax), %%mm5\n\t"
  93 "       pfadd   %%mm1, %%mm0\n\t"
  94 "       pfadd   %%mm5, %%mm4\n\t"
  95 "       movq    %%mm0, 48(%%edx)\n\t"
  96 "       movq    %%mm4, 56(%%edx)\n\t"
  97 "       pfsub   %%mm1, %%mm3\n\t"
  98 "       pfsub   %%mm5, %%mm7\n\t"
  99 "       pfmul   48(%%ebx), %%mm3\n\t"
 100 "       pfmul   56(%%ebx), %%mm7\n\t"
 101 "       pswapd  %%mm3, %%mm3\n\t"
 102 "       pswapd  %%mm7, %%mm7\n\t"
 103 "       movq    %%mm3, 72(%%edx)\n\t"
 104 "       movq    %%mm7, 64(%%edx)\n\t"
 105
 106 /* Phase 2*/
 107
 108 "       movq    (%%edx), %%mm0\n\t"
 109 "       movq    8(%%edx), %%mm4\n\t"
 110 "       movq    %%mm0, %%mm3\n\t"
 111 "       movq    %%mm4, %%mm7\n\t"
 112 "       pshufw  $78, 56(%%edx), %%mm1\n\t"
 113 "       pshufw  $78, 48(%%edx), %%mm5\n\t"
 114 "       pfadd   %%mm1, %%mm0\n\t"
 115 "       pfadd   %%mm5, %%mm4\n\t"
 116 "       movq    %%mm0, (%%ecx)\n\t"
 117 "       movq    %%mm4, 8(%%ecx)\n\t"
 118 "       pfsub   %%mm1, %%mm3\n\t"
 119 "       pfsub   %%mm5, %%mm7\n\t"
 120 "       pfmul   64(%%ebx), %%mm3\n\t"
 121 "       pfmul   72(%%ebx), %%mm7\n\t"
 122 "       pswapd  %%mm3, %%mm3\n\t"
 123 "       pswapd  %%mm7, %%mm7\n\t"
 124 "       movq    %%mm3, 56(%%ecx)\n\t"
 125 "       movq    %%mm7, 48(%%ecx)\n\t"
 126
 127 "       movq    16(%%edx), %%mm0\n\t"
 128 "       movq    24(%%edx), %%mm4\n\t"
 129 "       movq    %%mm0, %%mm3\n\t"
 130 "       movq    %%mm4, %%mm7\n\t"
 131 "       pshufw  $78, 40(%%edx), %%mm1\n\t"
 132 "       pshufw  $78, 32(%%edx), %%mm5\n\t"
 133 "       pfadd   %%mm1, %%mm0\n\t"
 134 "       pfadd   %%mm5, %%mm4\n\t"
 135 "       movq    %%mm0, 16(%%ecx)\n\t"
 136 "       movq    %%mm4, 24(%%ecx)\n\t"
 137 "       pfsub   %%mm1, %%mm3\n\t"
 138 "       pfsub   %%mm5, %%mm7\n\t"
 139 "       pfmul   80(%%ebx), %%mm3\n\t"
 140 "       pfmul   88(%%ebx), %%mm7\n\t"
 141 "       pswapd  %%mm3, %%mm3\n\t"
 142 "       pswapd  %%mm7, %%mm7\n\t"
 143 "       movq    %%mm3, 40(%%ecx)\n\t"
 144 "       movq    %%mm7, 32(%%ecx)\n\t"
 145
 146 /* Phase 3*/
 147
 148 "       movq    64(%%edx), %%mm0\n\t"
 149 "       movq    72(%%edx), %%mm4\n\t"
 150 "       movq    %%mm0, %%mm3\n\t"
 151 "       movq    %%mm4, %%mm7\n\t"
 152 "       pshufw  $78, 120(%%edx), %%mm1\n\t"
 153 "       pshufw  $78, 112(%%edx), %%mm5\n\t"
 154 "       pfadd   %%mm1, %%mm0\n\t"
 155 "       pfadd   %%mm5, %%mm4\n\t"
 156 "       movq    %%mm0, 64(%%ecx)\n\t"
 157 "       movq    %%mm4, 72(%%ecx)\n\t"
 158 "       pfsubr  %%mm1, %%mm3\n\t"
 159 "       pfsubr  %%mm5, %%mm7\n\t"
 160 "       pfmul   64(%%ebx), %%mm3\n\t"
 161 "       pfmul   72(%%ebx), %%mm7\n\t"
 162 "       pswapd  %%mm3, %%mm3\n\t"
 163 "       pswapd  %%mm7, %%mm7\n\t"
 164 "       movq    %%mm3, 120(%%ecx)\n\t"
 165 "       movq    %%mm7, 112(%%ecx)\n\t"
 166
 167 "       movq    80(%%edx), %%mm0\n\t"
 168 "       movq    88(%%edx), %%mm4\n\t"
 169 "       movq    %%mm0, %%mm3\n\t"
 170 "       movq    %%mm4, %%mm7\n\t"
 171 "       pshufw  $78, 104(%%edx), %%mm1\n\t"
 172 "       pshufw  $78, 96(%%edx), %%mm5\n\t"
 173 "       pfadd   %%mm1, %%mm0\n\t"
 174 "       pfadd   %%mm5, %%mm4\n\t"
 175 "       movq    %%mm0, 80(%%ecx)\n\t"
 176 "       movq    %%mm4, 88(%%ecx)\n\t"
 177 "       pfsubr  %%mm1, %%mm3\n\t"
 178 "       pfsubr  %%mm5, %%mm7\n\t"
 179 "       pfmul   80(%%ebx), %%mm3\n\t"
 180 "       pfmul   88(%%ebx), %%mm7\n\t"
 181 "       pswapd  %%mm3, %%mm3\n\t"
 182 "       pswapd  %%mm7, %%mm7\n\t"
 183 "       movq    %%mm3, 104(%%ecx)\n\t"
 184 "       movq    %%mm7, 96(%%ecx)\n\t"
 185
 186 /* Phase 4*/
 187
 188 "       movq    96(%%ebx), %%mm2\n\t"
 189 "       movq    104(%%ebx), %%mm6\n\t"
 190
 191 "       movq    (%%ecx), %%mm0\n\t"
 192 "       movq    8(%%ecx), %%mm4\n\t"
 193 "       movq    %%mm0, %%mm3\n\t"
 194 "       movq    %%mm4, %%mm7\n\t"
 195 "       pshufw  $78, 24(%%ecx), %%mm1\n\t"
 196 "       pshufw  $78, 16(%%ecx), %%mm5\n\t"
 197 "       pfadd   %%mm1, %%mm0\n\t"
 198 "       pfadd   %%mm5, %%mm4\n\t"
 199 "       movq    %%mm0, (%%edx)\n\t"
 200 "       movq    %%mm4, 8(%%edx)\n\t"
 201 "       pfsub   %%mm1, %%mm3\n\t"
 202 "       pfsub   %%mm5, %%mm7\n\t"
 203 "       pfmul   %%mm2, %%mm3\n\t"
 204 "       pfmul   %%mm6, %%mm7\n\t"
 205 "       pswapd  %%mm3, %%mm3\n\t"
 206 "       pswapd  %%mm7, %%mm7\n\t"
 207 "       movq    %%mm3, 24(%%edx)\n\t"
 208 "       movq    %%mm7, 16(%%edx)\n\t"
 209
 210 "       movq    32(%%ecx), %%mm0\n\t"
 211 "       movq    40(%%ecx), %%mm4\n\t"
 212 "       movq    %%mm0, %%mm3\n\t"
 213 "       movq    %%mm4, %%mm7\n\t"
 214 "       pshufw  $78, 56(%%ecx), %%mm1\n\t"
 215 "       pshufw  $78, 48(%%ecx), %%mm5\n\t"
 216 "       pfadd   %%mm1, %%mm0\n\t"
 217 "       pfadd   %%mm5, %%mm4\n\t"
 218 "       movq    %%mm0, 32(%%edx)\n\t"
 219 "       movq    %%mm4, 40(%%edx)\n\t"
 220 "       pfsubr  %%mm1, %%mm3\n\t"
 221 "       pfsubr  %%mm5, %%mm7\n\t"
 222 "       pfmul   %%mm2, %%mm3\n\t"
 223 "       pfmul   %%mm6, %%mm7\n\t"
 224 "       pswapd  %%mm3, %%mm3\n\t"
 225 "       pswapd  %%mm7, %%mm7\n\t"
 226 "       movq    %%mm3, 56(%%edx)\n\t"
 227 "       movq    %%mm7, 48(%%edx)\n\t"
 228
 229 "       movq    64(%%ecx), %%mm0\n\t"
 230 "       movq    72(%%ecx), %%mm4\n\t"
 231 "       movq    %%mm0, %%mm3\n\t"
 232 "       movq    %%mm4, %%mm7\n\t"
 233 "       pshufw  $78, 88(%%ecx), %%mm1\n\t"
 234 "       pshufw  $78, 80(%%ecx), %%mm5\n\t"
 235 "       pfadd   %%mm1, %%mm0\n\t"
 236 "       pfadd   %%mm5, %%mm4\n\t"
 237 "       movq    %%mm0, 64(%%edx)\n\t"
 238 "       movq    %%mm4, 72(%%edx)\n\t"
 239 "       pfsub   %%mm1, %%mm3\n\t"
 240 "       pfsub   %%mm5, %%mm7\n\t"
 241 "       pfmul   %%mm2, %%mm3\n\t"
 242 "       pfmul   %%mm6, %%mm7\n\t"
 243 "       pswapd  %%mm3, %%mm3\n\t"
 244 "       pswapd  %%mm7, %%mm7\n\t"
 245 "       movq    %%mm3, 88(%%edx)\n\t"
 246 "       movq    %%mm7, 80(%%edx)\n\t"
 247
 248 "       movq    96(%%ecx), %%mm0\n\t"
 249 "       movq    104(%%ecx), %%mm4\n\t"
 250 "       movq    %%mm0, %%mm3\n\t"
 251 "       movq    %%mm4, %%mm7\n\t"
 252 "       pshufw  $78, 120(%%ecx), %%mm1\n\t"
 253 "       pshufw  $78, 112(%%ecx), %%mm5\n\t"
 254 "       pfadd   %%mm1, %%mm0\n\t"
 255 "       pfadd   %%mm5, %%mm4\n\t"
 256 "       movq    %%mm0, 96(%%edx)\n\t"
 257 "       movq    %%mm4, 104(%%edx)\n\t"
 258 "       pfsubr  %%mm1, %%mm3\n\t"
 259 "       pfsubr  %%mm5, %%mm7\n\t"
 260 "       pfmul   %%mm2, %%mm3\n\t"
 261 "       pfmul   %%mm6, %%mm7\n\t"
 262 "       pswapd  %%mm3, %%mm3\n\t"
 263 "       pswapd  %%mm7, %%mm7\n\t"
 264 "       movq    %%mm3, 120(%%edx)\n\t"
 265 "       movq    %%mm7, 112(%%edx)\n\t"
 266
 267 /* Phase 5 */
 268
 269 "       movq    112(%%ebx), %%mm2\n\t"
 270
 271 "       movq    (%%edx), %%mm0\n\t"
 272 "       movq    16(%%edx), %%mm4\n\t"
 273 "       movq    %%mm0, %%mm3\n\t"
 274 "       movq    %%mm4, %%mm7\n\t"
 275 "       pshufw  $78, 8(%%edx), %%mm1\n\t"
 276 "       pshufw  $78, 24(%%edx), %%mm5\n\t"
 277 "       pfadd   %%mm1, %%mm0\n\t"
 278 "       pfadd   %%mm5, %%mm4\n\t"
 279 "       movq    %%mm0, (%%ecx)\n\t"
 280 "       movq    %%mm4, 16(%%ecx)\n\t"
 281 "       pfsub   %%mm1, %%mm3\n\t"
 282 "       pfsubr  %%mm5, %%mm7\n\t"
 283 "       pfmul   %%mm2, %%mm3\n\t"
 284 "       pfmul   %%mm2, %%mm7\n\t"
 285 "       pswapd  %%mm3, %%mm3\n\t"
 286 "       pswapd  %%mm7, %%mm7\n\t"
 287 "       movq    %%mm3, 8(%%ecx)\n\t"
 288 "       movq    %%mm7, 24(%%ecx)\n\t"
 289
 290 "       movq    32(%%edx), %%mm0\n\t"
 291 "       movq    48(%%edx), %%mm4\n\t"
 292 "       movq    %%mm0, %%mm3\n\t"
 293 "       movq    %%mm4, %%mm7\n\t"
 294 "       pshufw  $78, 40(%%edx), %%mm1\n\t"
 295 "       pshufw  $78, 56(%%edx), %%mm5\n\t"
 296 "       pfadd   %%mm1, %%mm0\n\t"
 297 "       pfadd   %%mm5, %%mm4\n\t"
 298 "       movq    %%mm0, 32(%%ecx)\n\t"
 299 "       movq    %%mm4, 48(%%ecx)\n\t"
 300 "       pfsub   %%mm1, %%mm3\n\t"
 301 "       pfsubr  %%mm5, %%mm7\n\t"
 302 "       pfmul   %%mm2, %%mm3\n\t"
 303 "       pfmul   %%mm2, %%mm7\n\t"
 304 "       pswapd  %%mm3, %%mm3\n\t"
 305 "       pswapd  %%mm7, %%mm7\n\t"
 306 "       movq    %%mm3, 40(%%ecx)\n\t"
 307 "       movq    %%mm7, 56(%%ecx)\n\t"
 308
 309 "       movq    64(%%edx), %%mm0\n\t"
 310 "       movq    80(%%edx), %%mm4\n\t"
 311 "       movq    %%mm0, %%mm3\n\t"
 312 "       movq    %%mm4, %%mm7\n\t"
 313 "       pshufw  $78, 72(%%edx), %%mm1\n\t"
 314 "       pshufw  $78, 88(%%edx), %%mm5\n\t"
 315 "       pfadd   %%mm1, %%mm0\n\t"
 316 "       pfadd   %%mm5, %%mm4\n\t"
 317 "       movq    %%mm0, 64(%%ecx)\n\t"
 318 "       movq    %%mm4, 80(%%ecx)\n\t"
 319 "       pfsub   %%mm1, %%mm3\n\t"
 320 "       pfsubr  %%mm5, %%mm7\n\t"
 321 "       pfmul   %%mm2, %%mm3\n\t"
 322 "       pfmul   %%mm2, %%mm7\n\t"
 323 "       pswapd  %%mm3, %%mm3\n\t"
 324 "       pswapd  %%mm7, %%mm7\n\t"
 325 "       movq    %%mm3, 72(%%ecx)\n\t"
 326 "       movq    %%mm7, 88(%%ecx)\n\t"
 327
 328 "       movq    96(%%edx), %%mm0\n\t"
 329 "       movq    112(%%edx), %%mm4\n\t"
 330 "       movq    %%mm0, %%mm3\n\t"
 331 "       movq    %%mm4, %%mm7\n\t"
 332 "       pshufw  $78, 104(%%edx), %%mm1\n\t"
 333 "       pshufw  $78, 120(%%edx), %%mm5\n\t"
 334 "       pfadd   %%mm1, %%mm0\n\t"
 335 "       pfadd   %%mm5, %%mm4\n\t"
 336 "       movq    %%mm0, 96(%%ecx)\n\t"
 337 "       movq    %%mm4, 112(%%ecx)\n\t"
 338 "       pfsub   %%mm1, %%mm3\n\t"
 339 "       pfsubr  %%mm5, %%mm7\n\t"
 340 "       pfmul   %%mm2, %%mm3\n\t"
 341 "       pfmul   %%mm2, %%mm7\n\t"
 342 "       pswapd  %%mm3, %%mm3\n\t"
 343 "       pswapd  %%mm7, %%mm7\n\t"
 344 "       movq    %%mm3, 104(%%ecx)\n\t"
 345 "       movq    %%mm7, 120(%%ecx)\n\t"
 346
 347
 348 /* Phase 6. This is the end of easy road. */
 349 /* Code below is coded in scalar mode. Should be optimized */
 350
 351 "       movd    "MANGLE(plus_1f)", %%mm6\n\t"
 352 "       punpckldq 120(%%ebx), %%mm6\n\t"      /* mm6 = 1.0 | 120(%%ebx)*/
 353 "       movq    "MANGLE(x_plus_minus_3dnow)", %%mm7\n\t" /* mm7 = +1 | -1 */
 354
 355 "       movq    32(%%ecx), %%mm0\n\t"
 356 "       movq    64(%%ecx), %%mm2\n\t"
 357 "       movq    %%mm0, %%mm1\n\t"
 358 "       movq    %%mm2, %%mm3\n\t"
 359 "       pxor    %%mm7, %%mm1\n\t"
 360 "       pxor    %%mm7, %%mm3\n\t"
 361 "       pfacc   %%mm1, %%mm0\n\t"
 362 "       pfacc   %%mm3, %%mm2\n\t"
 363 "       pfmul   %%mm6, %%mm0\n\t"
 364 "       pfmul   %%mm6, %%mm2\n\t"
 365 "       movq    %%mm0, 32(%%edx)\n\t"
 366 "       movq    %%mm2, 64(%%edx)\n\t"
 367
 368 "       movd    44(%%ecx), %%mm0\n\t"
 369 "       movd    40(%%ecx), %%mm2\n\t"
 370 "       movd    120(%%ebx), %%mm3\n\t"
 371 "       punpckldq 76(%%ecx), %%mm0\n\t"
 372 "       punpckldq 72(%%ecx), %%mm2\n\t"
 373 "       punpckldq %%mm3, %%mm3\n\t"
 374 "       movq    %%mm0, %%mm4\n\t"
 375 "       movq    %%mm2, %%mm5\n\t"
 376 "       pfsub   %%mm2, %%mm0\n\t"
 377 "       pfmul   %%mm3, %%mm0\n\t"
 378 "       movq    %%mm0, %%mm1\n\t"
 379 "       pfadd   %%mm5, %%mm0\n\t"
 380 "       pfadd   %%mm4, %%mm0\n\t"
 381 "       movq    %%mm0, %%mm2\n\t"
 382 "       punpckldq %%mm1, %%mm0\n\t"
 383 "       punpckhdq %%mm1, %%mm2\n\t"
 384 "       movq    %%mm0, 40(%%edx)\n\t"
 385 "       movq    %%mm2, 72(%%edx)\n\t"
 386
 387 "       movd   48(%%ecx), %%mm3\n\t"
 388 "       movd   60(%%ecx), %%mm2\n\t"
 389 "       pfsub  52(%%ecx), %%mm3\n\t"
 390 "       pfsub  56(%%ecx), %%mm2\n\t"
 391 "       pfmul 120(%%ebx), %%mm3\n\t"
 392 "       pfmul 120(%%ebx), %%mm2\n\t"
 393 "       movq    %%mm2, %%mm1\n\t"
 394
 395 "       pfadd  56(%%ecx), %%mm1\n\t"
 396 "       pfadd  60(%%ecx), %%mm1\n\t"
 397 "       movq    %%mm1, %%mm0\n\t"
 398
 399 "       pfadd  48(%%ecx), %%mm0\n\t"
 400 "       pfadd  52(%%ecx), %%mm0\n\t"
 401 "       pfadd   %%mm3, %%mm1\n\t"
 402 "       punpckldq %%mm2, %%mm1\n\t"
 403 "       pfadd   %%mm3, %%mm2\n\t"
 404 "       punpckldq %%mm2, %%mm0\n\t"
 405 "       movq    %%mm1, 56(%%edx)\n\t"
 406 "       movq    %%mm0, 48(%%edx)\n\t"
 407
 408 /*---*/
 409
 410 "       movd   92(%%ecx), %%mm1\n\t"
 411 "       pfsub  88(%%ecx), %%mm1\n\t"
 412 "       pfmul 120(%%ebx), %%mm1\n\t"
 413 "       movd   %%mm1, 92(%%edx)\n\t"
 414 "       pfadd  92(%%ecx), %%mm1\n\t"
 415 "       pfadd  88(%%ecx), %%mm1\n\t"
 416 "       movq   %%mm1, %%mm0\n\t"
 417
 418 "       pfadd  80(%%ecx), %%mm0\n\t"
 419 "       pfadd  84(%%ecx), %%mm0\n\t"
 420 "       movd   %%mm0, 80(%%edx)\n\t"
 421
 422 "       movd   80(%%ecx), %%mm0\n\t"
 423 "       pfsub  84(%%ecx), %%mm0\n\t"
 424 "       pfmul 120(%%ebx), %%mm0\n\t"
 425 "       pfadd  %%mm0, %%mm1\n\t"
 426 "       pfadd  92(%%edx), %%mm0\n\t"
 427 "       punpckldq %%mm1, %%mm0\n\t"
 428 "       movq   %%mm0, 84(%%edx)\n\t"
 429
 430 "       movq    96(%%ecx), %%mm0\n\t"
 431 "       movq    %%mm0, %%mm1\n\t"
 432 "       pxor    %%mm7, %%mm1\n\t"
 433 "       pfacc   %%mm1, %%mm0\n\t"
 434 "       pfmul   %%mm6, %%mm0\n\t"
 435 "       movq    %%mm0, 96(%%edx)\n\t"
 436
 437 "       movd  108(%%ecx), %%mm0\n\t"
 438 "       pfsub 104(%%ecx), %%mm0\n\t"
 439 "       pfmul 120(%%ebx), %%mm0\n\t"
 440 "       movd  %%mm0, 108(%%edx)\n\t"
 441 "       pfadd 104(%%ecx), %%mm0\n\t"
 442 "       pfadd 108(%%ecx), %%mm0\n\t"
 443 "       movd  %%mm0, 104(%%edx)\n\t"
 444
 445 "       movd  124(%%ecx), %%mm1\n\t"
 446 "       pfsub 120(%%ecx), %%mm1\n\t"
 447 "       pfmul 120(%%ebx), %%mm1\n\t"
 448 "       movd  %%mm1, 124(%%edx)\n\t"
 449 "       pfadd 120(%%ecx), %%mm1\n\t"
 450 "       pfadd 124(%%ecx), %%mm1\n\t"
 451 "       movq  %%mm1, %%mm0\n\t"
 452
 453 "       pfadd 112(%%ecx), %%mm0\n\t"
 454 "       pfadd 116(%%ecx), %%mm0\n\t"
 455 "       movd  %%mm0, 112(%%edx)\n\t"
 456
 457 "       movd  112(%%ecx), %%mm0\n\t"
 458 "       pfsub 116(%%ecx), %%mm0\n\t"
 459 "       pfmul 120(%%ebx), %%mm0\n\t"
 460 "       pfadd %%mm0,%%mm1\n\t"
 461 "       pfadd 124(%%edx), %%mm0\n\t"
 462 "       punpckldq %%mm1, %%mm0\n\t"
 463 "       movq  %%mm0, 116(%%edx)\n\t"
 464
 465 "       jnz .L01\n\t"
 466
 467 /* Phase 7*/
 468 /* Code below is coded in scalar mode. Should be optimized */
 469
 470 "       movd      (%%ecx), %%mm0\n\t"
 471 "       pfadd    4(%%ecx), %%mm0\n\t"
 472 "       movd     %%mm0, 1024(%%esi)\n\t"
 473
 474 "       movd      (%%ecx), %%mm0\n\t"
 475 "       pfsub    4(%%ecx), %%mm0\n\t"
 476 "       pfmul  120(%%ebx), %%mm0\n\t"
 477 "       movd      %%mm0, (%%esi)\n\t"
 478 "       movd      %%mm0, (%%edi)\n\t"
 479
 480 "       movd   12(%%ecx), %%mm0\n\t"
 481 "       pfsub   8(%%ecx), %%mm0\n\t"
 482 "       pfmul 120(%%ebx), %%mm0\n\t"
 483 "       movd    %%mm0, 512(%%edi)\n\t"
 484 "       pfadd   12(%%ecx), %%mm0\n\t"
 485 "       pfadd   8(%%ecx), %%mm0\n\t"
 486 "       movd    %%mm0, 512(%%esi)\n\t"
 487
 488 "       movd   16(%%ecx), %%mm0\n\t"
 489 "       pfsub  20(%%ecx), %%mm0\n\t"
 490 "       pfmul 120(%%ebx), %%mm0\n\t"
 491 "       movq    %%mm0, %%mm3\n\t"
 492
 493 "       movd   28(%%ecx), %%mm0\n\t"
 494 "       pfsub  24(%%ecx), %%mm0\n\t"
 495 "       pfmul 120(%%ebx), %%mm0\n\t"
 496 "       movd    %%mm0, 768(%%edi)\n\t"
 497 "       movq    %%mm0, %%mm2\n\t"
 498
 499 "       pfadd  24(%%ecx), %%mm0\n\t"
 500 "       pfadd  28(%%ecx), %%mm0\n\t"
 501 "       movq    %%mm0, %%mm1\n\t"
 502
 503 "       pfadd  16(%%ecx), %%mm0\n\t"
 504 "       pfadd  20(%%ecx), %%mm0\n\t"
 505 "       movd   %%mm0, 768(%%esi)\n\t"
 506 "       pfadd  %%mm3, %%mm1\n\t"
 507 "       movd   %%mm1, 256(%%esi)\n\t"
 508 "       pfadd  %%mm3, %%mm2\n\t"
 509 "       movd   %%mm2, 256(%%edi)\n\t"
 510
 511 /* Phase 8*/
 512
 513 "       movq   32(%%edx), %%mm0\n\t"
 514 "       movq   48(%%edx), %%mm1\n\t"
 515 "       pfadd  48(%%edx), %%mm0\n\t"
 516 "       pfadd  40(%%edx), %%mm1\n\t"
 517 "       movd   %%mm0, 896(%%esi)\n\t"
 518 "       movd   %%mm1, 640(%%esi)\n\t"
 519 "       psrlq  $32, %%mm0\n\t"
 520 "       psrlq  $32, %%mm1\n\t"
 521 "       movd   %%mm0, 128(%%edi)\n\t"
 522 "       movd   %%mm1, 384(%%edi)\n\t"
 523
 524 "       movd   40(%%edx), %%mm0\n\t"
 525 "       pfadd  56(%%edx), %%mm0\n\t"
 526 "       movd   %%mm0, 384(%%esi)\n\t"
 527
 528 "       movd   56(%%edx), %%mm0\n\t"
 529 "       pfadd  36(%%edx), %%mm0\n\t"
 530 "       movd   %%mm0, 128(%%esi)\n\t"
 531
 532 "       movd   60(%%edx), %%mm0\n\t"
 533 "       movd   %%mm0, 896(%%edi)\n\t"
 534 "       pfadd  44(%%edx), %%mm0\n\t"
 535 "       movd   %%mm0, 640(%%edi)\n\t"
 536
 537 "       movq   96(%%edx), %%mm0\n\t"
 538 "       movq   112(%%edx), %%mm2\n\t"
 539 "       movq   104(%%edx), %%mm4\n\t"
 540 "       pfadd  112(%%edx), %%mm0\n\t"
 541 "       pfadd  104(%%edx), %%mm2\n\t"
 542 "       pfadd  120(%%edx), %%mm4\n\t"
 543 "       movq   %%mm0, %%mm1\n\t"
 544 "       movq   %%mm2, %%mm3\n\t"
 545 "       movq   %%mm4, %%mm5\n\t"
 546 "       pfadd  64(%%edx), %%mm0\n\t"
 547 "       pfadd  80(%%edx), %%mm2\n\t"
 548 "       pfadd  72(%%edx), %%mm4\n\t"
 549 "       movd   %%mm0, 960(%%esi)\n\t"
 550 "       movd   %%mm2, 704(%%esi)\n\t"
 551 "       movd   %%mm4, 448(%%esi)\n\t"
 552 "       psrlq  $32, %%mm0\n\t"
 553 "       psrlq  $32, %%mm2\n\t"
 554 "       psrlq  $32, %%mm4\n\t"
 555 "       movd   %%mm0, 64(%%edi)\n\t"
 556 "       movd   %%mm2, 320(%%edi)\n\t"
 557 "       movd   %%mm4, 576(%%edi)\n\t"
 558 "       pfadd  80(%%edx), %%mm1\n\t"
 559 "       pfadd  72(%%edx), %%mm3\n\t"
 560 "       pfadd  88(%%edx), %%mm5\n\t"
 561 "       movd   %%mm1, 832(%%esi)\n\t"
 562 "       movd   %%mm3, 576(%%esi)\n\t"
 563 "       movd   %%mm5, 320(%%esi)\n\t"
 564 "       psrlq  $32, %%mm1\n\t"
 565 "       psrlq  $32, %%mm3\n\t"
 566 "       psrlq  $32, %%mm5\n\t"
 567 "       movd   %%mm1, 192(%%edi)\n\t"
 568 "       movd   %%mm3, 448(%%edi)\n\t"
 569 "       movd   %%mm5, 704(%%edi)\n\t"
 570
 571 "       movd   120(%%edx), %%mm0\n\t"
 572 "       pfadd  100(%%edx), %%mm0\n\t"
 573 "       movq   %%mm0, %%mm1\n\t"
 574 "       pfadd  88(%%edx), %%mm0\n\t"
 575 "       movd   %%mm0, 192(%%esi)\n\t"
 576 "       pfadd  68(%%edx), %%mm1\n\t"
 577 "       movd   %%mm1, 64(%%esi)\n\t"
 578
 579 "       movd  124(%%edx), %%mm0\n\t"
 580 "       movd  %%mm0, 960(%%edi)\n\t"
 581 "       pfadd  92(%%edx), %%mm0\n\t"
 582 "       movd  %%mm0, 832(%%edi)\n\t"
 583
 584 "       jmp     .L_bye\n\t"
 585 ".L01:  \n\t"
 586 /* Phase 9*/
 587
 588 "       movq    (%%ecx), %%mm0\n\t"
 589 "       movq    %%mm0, %%mm1\n\t"
 590 "       pxor    %%mm7, %%mm1\n\t"
 591 "       pfacc   %%mm1, %%mm0\n\t"
 592 "       pfmul   %%mm6, %%mm0\n\t"
 593 "       pf2id   %%mm0, %%mm0\n\t"
 594 "       movd    %%mm0, %%eax\n\t"
 595 "       movw    %%ax, 512(%%esi)\n\t"
 596 "       psrlq   $32, %%mm0\n\t"
 597 "       movd    %%mm0, %%eax\n\t"
 598 "       movw    %%ax, (%%esi)\n\t"
 599
 600 "       movd    12(%%ecx), %%mm0\n\t"
 601 "       pfsub    8(%%ecx), %%mm0\n\t"
 602 "       pfmul  120(%%ebx), %%mm0\n\t"
 603 "       pf2id    %%mm0, %%mm7\n\t"
 604 "       movd     %%mm7, %%eax\n\t"
 605 "       movw     %%ax, 256(%%edi)\n\t"
 606 "       pfadd   12(%%ecx), %%mm0\n\t"
 607 "       pfadd    8(%%ecx), %%mm0\n\t"
 608 "       pf2id    %%mm0, %%mm0\n\t"
 609 "       movd     %%mm0, %%eax\n\t"
 610 "       movw     %%ax, 256(%%esi)\n\t"
 611
 612 "       movd   16(%%ecx), %%mm3\n\t"
 613 "       pfsub  20(%%ecx), %%mm3\n\t"
 614 "       pfmul  120(%%ebx), %%mm3\n\t"
 615 "       movq   %%mm3, %%mm2\n\t"
 616
 617 "       movd   28(%%ecx), %%mm2\n\t"
 618 "       pfsub  24(%%ecx), %%mm2\n\t"
 619 "       pfmul 120(%%ebx), %%mm2\n\t"
 620 "       movq   %%mm2, %%mm1\n\t"
 621
 622 "       pf2id  %%mm2, %%mm7\n\t"
 623 "       movd   %%mm7, %%eax\n\t"
 624 "       movw   %%ax, 384(%%edi)\n\t"
 625
 626 "       pfadd  24(%%ecx), %%mm1\n\t"
 627 "       pfadd  28(%%ecx), %%mm1\n\t"
 628 "       movq   %%mm1, %%mm0\n\t"
 629
 630 "       pfadd  16(%%ecx), %%mm0\n\t"
 631 "       pfadd  20(%%ecx), %%mm0\n\t"
 632 "       pf2id  %%mm0, %%mm0\n\t"
 633 "       movd   %%mm0, %%eax\n\t"
 634 "       movw   %%ax, 384(%%esi)\n\t"
 635 "       pfadd  %%mm3, %%mm1\n\t"
 636 "       pf2id  %%mm1, %%mm1\n\t"
 637 "       movd   %%mm1, %%eax\n\t"
 638 "       movw   %%ax, 128(%%esi)\n\t"
 639 "       pfadd  %%mm3, %%mm2\n\t"
 640 "       pf2id  %%mm2, %%mm2\n\t"
 641 "       movd   %%mm2, %%eax\n\t"
 642 "       movw   %%ax, 128(%%edi)\n\t"
 643
 644 /* Phase 10*/
 645
 646 "       movq    32(%%edx), %%mm0\n\t"
 647 "       movq    48(%%edx), %%mm1\n\t"
 648 "       pfadd   48(%%edx), %%mm0\n\t"
 649 "       pfadd   40(%%edx), %%mm1\n\t"
 650 "       pf2id   %%mm0, %%mm0\n\t"
 651 "       pf2id   %%mm1, %%mm1\n\t"
 652 "       movd    %%mm0, %%eax\n\t"
 653 "       movd    %%mm1, %%ecx\n\t"
 654 "       movw    %%ax, 448(%%esi)\n\t"
 655 "       movw    %%cx, 320(%%esi)\n\t"
 656 "       psrlq   $32, %%mm0\n\t"
 657 "       psrlq   $32, %%mm1\n\t"
 658 "       movd    %%mm0, %%eax\n\t"
 659 "       movd    %%mm1, %%ecx\n\t"
 660 "       movw    %%ax, 64(%%edi)\n\t"
 661 "       movw    %%cx, 192(%%edi)\n\t"
 662
 663 "       movd   40(%%edx), %%mm3\n\t"
 664 "       movd   56(%%edx), %%mm4\n\t"
 665 "       movd   60(%%edx), %%mm0\n\t"
 666 "       movd   44(%%edx), %%mm2\n\t"
 667 "       movd  120(%%edx), %%mm5\n\t"
 668 "       punpckldq %%mm4, %%mm3\n\t"
 669 "       punpckldq 124(%%edx), %%mm0\n\t"
 670 "       pfadd 100(%%edx), %%mm5\n\t"
 671 "       punpckldq 36(%%edx), %%mm4\n\t"
 672 "       punpckldq 92(%%edx), %%mm2\n\t"
 673 "       movq  %%mm5, %%mm6\n\t"
 674 "       pfadd  %%mm4, %%mm3\n\t"
 675 "       pf2id  %%mm0, %%mm1\n\t"
 676 "       pf2id  %%mm3, %%mm3\n\t"
 677 "       pfadd  88(%%edx), %%mm5\n\t"
 678 "       movd   %%mm1, %%eax\n\t"
 679 "       movd   %%mm3, %%ecx\n\t"
 680 "       movw   %%ax, 448(%%edi)\n\t"
 681 "       movw   %%cx, 192(%%esi)\n\t"
 682 "       pf2id  %%mm5, %%mm5\n\t"
 683 "       psrlq  $32, %%mm1\n\t"
 684 "       psrlq  $32, %%mm3\n\t"
 685 "       movd   %%mm5, %%ebx\n\t"
 686 "       movd   %%mm1, %%eax\n\t"
 687 "       movd   %%mm3, %%ecx\n\t"
 688 "       movw   %%bx, 96(%%esi)\n\t"
 689 "       movw   %%ax, 480(%%edi)\n\t"
 690 "       movw   %%cx, 64(%%esi)\n\t"
 691 "       pfadd  %%mm2, %%mm0\n\t"
 692 "       pf2id  %%mm0, %%mm0\n\t"
 693 "       movd   %%mm0, %%eax\n\t"
 694 "       pfadd  68(%%edx), %%mm6\n\t"
 695 "       movw   %%ax, 320(%%edi)\n\t"
 696 "       psrlq  $32, %%mm0\n\t"
 697 "       pf2id  %%mm6, %%mm6\n\t"
 698 "       movd   %%mm0, %%eax\n\t"
 699 "       movd   %%mm6, %%ebx\n\t"
 700 "       movw   %%ax, 416(%%edi)\n\t"
 701 "       movw   %%bx, 32(%%esi)\n\t"
 702
 703 "       movq   96(%%edx), %%mm0\n\t"
 704 "       movq  112(%%edx), %%mm2\n\t"
 705 "       movq  104(%%edx), %%mm4\n\t"
 706 "       pfadd %%mm2, %%mm0\n\t"
 707 "       pfadd %%mm4, %%mm2\n\t"
 708 "       pfadd 120(%%edx), %%mm4\n\t"
 709 "       movq  %%mm0, %%mm1\n\t"
 710 "       movq  %%mm2, %%mm3\n\t"
 711 "       movq  %%mm4, %%mm5\n\t"
 712 "       pfadd  64(%%edx), %%mm0\n\t"
 713 "       pfadd  80(%%edx), %%mm2\n\t"
 714 "       pfadd  72(%%edx), %%mm4\n\t"
 715 "       pf2id  %%mm0, %%mm0\n\t"
 716 "       pf2id  %%mm2, %%mm2\n\t"
 717 "       pf2id  %%mm4, %%mm4\n\t"
 718 "       movd   %%mm0, %%eax\n\t"
 719 "       movd   %%mm2, %%ecx\n\t"
 720 "       movd   %%mm4, %%ebx\n\t"
 721 "       movw   %%ax, 480(%%esi)\n\t"
 722 "       movw   %%cx, 352(%%esi)\n\t"
 723 "       movw   %%bx, 224(%%esi)\n\t"
 724 "       psrlq  $32, %%mm0\n\t"
 725 "       psrlq  $32, %%mm2\n\t"
 726 "       psrlq  $32, %%mm4\n\t"
 727 "       movd   %%mm0, %%eax\n\t"
 728 "       movd   %%mm2, %%ecx\n\t"
 729 "       movd   %%mm4, %%ebx\n\t"
 730 "       movw   %%ax, 32(%%edi)\n\t"
 731 "       movw   %%cx, 160(%%edi)\n\t"
 732 "       movw   %%bx, 288(%%edi)\n\t"
 733 "       pfadd  80(%%edx), %%mm1\n\t"
 734 "       pfadd  72(%%edx), %%mm3\n\t"
 735 "       pfadd  88(%%edx), %%mm5\n\t"
 736 "       pf2id  %%mm1, %%mm1\n\t"
 737 "       pf2id  %%mm3, %%mm3\n\t"
 738 "       pf2id  %%mm5, %%mm5\n\t"
 739 "       movd   %%mm1, %%eax\n\t"
 740 "       movd   %%mm3, %%ecx\n\t"
 741 "       movd   %%mm5, %%ebx\n\t"
 742 "       movw   %%ax, 416(%%esi)\n\t"
 743 "       movw   %%cx, 288(%%esi)\n\t"
 744 "       movw   %%bx, 160(%%esi)\n\t"
 745 "       psrlq  $32, %%mm1\n\t"
 746 "       psrlq  $32, %%mm3\n\t"
 747 "       psrlq  $32, %%mm5\n\t"
 748 "       movd   %%mm1, %%eax\n\t"
 749 "       movd   %%mm3, %%ecx\n\t"
 750 "       movd   %%mm5, %%ebx\n\t"
 751 "       movw   %%ax, 96(%%edi)\n\t"
 752 "       movw   %%cx, 224(%%edi)\n\t"
 753 "       movw   %%bx, 352(%%edi)\n\t"
 754
 755 "       movsw\n\t"
 756
 757 ".L_bye:\n\t"
 758 "       femms\n\t"
 759         :
 760         :"m"(a),"m"(b),"m"(c),"m"(tmp[0])
 761         :"memory","%eax","%ebx","%ecx","%edx","%esi","%edi");
 762 }