libavcodec/ppc/h264_template_altivec.c

   1 /*
   2  * Copyright (c) 2004 Romain Dolbeau <romain@dolbeau.org>
   3  *
   4  * This file is part of FFmpeg.
   5  *
   6  * FFmpeg is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10  *
  11  * FFmpeg is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with FFmpeg; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  19  */
  20
  21 //#define DEBUG_ALIGNMENT
  22 #ifdef DEBUG_ALIGNMENT
  23 #define ASSERT_ALIGNED(ptr) assert(((unsigned long)ptr&0x0000000F));
  24 #else
  25 #define ASSERT_ALIGNED(ptr) ;
  26 #endif
  27
  28 /* this code assume that stride % 16 == 0 */
  29
  30 #define CHROMA_MC8_ALTIVEC_CORE \
  31         vsrc2ssH = (vec_s16_t)vec_mergeh(zero_u8v,(vec_u8_t)vsrc2uc);\
  32         vsrc3ssH = (vec_s16_t)vec_mergeh(zero_u8v,(vec_u8_t)vsrc3uc);\
  33 \
  34         psum = vec_mladd(vA, vsrc0ssH, v32ss);\
  35         psum = vec_mladd(vB, vsrc1ssH, psum);\
  36         psum = vec_mladd(vC, vsrc2ssH, psum);\
  37         psum = vec_mladd(vD, vsrc3ssH, psum);\
  38         psum = vec_sr(psum, v6us);\
  39 \
  40         vdst = vec_ld(0, dst);\
  41         ppsum = (vec_u8_t)vec_pack(psum, psum);\
  42         vfdst = vec_perm(vdst, ppsum, fperm);\
  43 \
  44         OP_U8_ALTIVEC(fsum, vfdst, vdst);\
  45 \
  46         vec_st(fsum, 0, dst);\
  47 \
  48         vsrc0ssH = vsrc2ssH;\
  49         vsrc1ssH = vsrc3ssH;\
  50 \
  51         dst += stride;\
  52         src += stride;
  53
  54 #define CHROMA_MC8_ALTIVEC_CORE_SIMPLE \
  55 \
  56         vsrc0ssH = (vec_s16_t)vec_mergeh(zero_u8v,(vec_u8_t)vsrc0uc);\
  57         vsrc1ssH = (vec_s16_t)vec_mergeh(zero_u8v,(vec_u8_t)vsrc1uc);\
  58 \
  59         psum = vec_mladd(vA, vsrc0ssH, v32ss);\
  60         psum = vec_mladd(vE, vsrc1ssH, psum);\
  61         psum = vec_sr(psum, v6us);\
  62 \
  63         vdst = vec_ld(0, dst);\
  64         ppsum = (vec_u8_t)vec_pack(psum, psum);\
  65         vfdst = vec_perm(vdst, ppsum, fperm);\
  66 \
  67         OP_U8_ALTIVEC(fsum, vfdst, vdst);\
  68 \
  69         vec_st(fsum, 0, dst);\
  70 \
  71         dst += stride;\
  72         src += stride;
  73
  74 void PREFIX_h264_chroma_mc8_altivec(uint8_t * dst, uint8_t * src,
  75                                     int stride, int h, int x, int y) {
  76   POWERPC_PERF_DECLARE(PREFIX_h264_chroma_mc8_num, 1);
  77     DECLARE_ALIGNED_16(signed int, ABCD[4]) =
  78                         {((8 - x) * (8 - y)),
  79                          ((    x) * (8 - y)),
  80                          ((8 - x) * (    y)),
  81                          ((    x) * (    y))};
  82     register int i;
  83     vec_u8_t fperm;
  84     const vec_s32_t vABCD = vec_ld(0, ABCD);
  85     const vec_s16_t vA = vec_splat((vec_s16_t)vABCD, 1);
  86     const vec_s16_t vB = vec_splat((vec_s16_t)vABCD, 3);
  87     const vec_s16_t vC = vec_splat((vec_s16_t)vABCD, 5);
  88     const vec_s16_t vD = vec_splat((vec_s16_t)vABCD, 7);
  89     LOAD_ZERO;
  90     const vec_s16_t v32ss = vec_sl(vec_splat_s16(1),vec_splat_u16(5));
  91     const vec_u16_t v6us = vec_splat_u16(6);
  92     register int loadSecond = (((unsigned long)src) % 16) <= 7 ? 0 : 1;
  93     register int reallyBadAlign = (((unsigned long)src) % 16) == 15 ? 1 : 0;
  94
  95     vec_u8_t vsrcAuc, vsrcBuc, vsrcperm0, vsrcperm1;
  96     vec_u8_t vsrc0uc, vsrc1uc;
  97     vec_s16_t vsrc0ssH, vsrc1ssH;
  98     vec_u8_t vsrcCuc, vsrc2uc, vsrc3uc;
  99     vec_s16_t vsrc2ssH, vsrc3ssH, psum;
 100     vec_u8_t vdst, ppsum, vfdst, fsum;
 101
 102   POWERPC_PERF_START_COUNT(PREFIX_h264_chroma_mc8_num, 1);
 103
 104     if (((unsigned long)dst) % 16 == 0) {
 105         fperm = (vec_u8_t){0x10, 0x11, 0x12, 0x13,
 106                            0x14, 0x15, 0x16, 0x17,
 107                            0x08, 0x09, 0x0A, 0x0B,
 108                            0x0C, 0x0D, 0x0E, 0x0F};
 109     } else {
 110         fperm = (vec_u8_t){0x00, 0x01, 0x02, 0x03,
 111                            0x04, 0x05, 0x06, 0x07,
 112                            0x18, 0x19, 0x1A, 0x1B,
 113                            0x1C, 0x1D, 0x1E, 0x1F};
 114     }
 115
 116     vsrcAuc = vec_ld(0, src);
 117
 118     if (loadSecond)
 119         vsrcBuc = vec_ld(16, src);
 120     vsrcperm0 = vec_lvsl(0, src);
 121     vsrcperm1 = vec_lvsl(1, src);
 122
 123     vsrc0uc = vec_perm(vsrcAuc, vsrcBuc, vsrcperm0);
 124     if (reallyBadAlign)
 125         vsrc1uc = vsrcBuc;
 126     else
 127         vsrc1uc = vec_perm(vsrcAuc, vsrcBuc, vsrcperm1);
 128
 129     vsrc0ssH = (vec_s16_t)vec_mergeh(zero_u8v,(vec_u8_t)vsrc0uc);
 130     vsrc1ssH = (vec_s16_t)vec_mergeh(zero_u8v,(vec_u8_t)vsrc1uc);
 131
 132     if (ABCD[3]) {
 133         if (!loadSecond) {// -> !reallyBadAlign
 134             for (i = 0 ; i < h ; i++) {
 135                 vsrcCuc = vec_ld(stride + 0, src);
 136                 vsrc2uc = vec_perm(vsrcCuc, vsrcCuc, vsrcperm0);
 137                 vsrc3uc = vec_perm(vsrcCuc, vsrcCuc, vsrcperm1);
 138
 139                 CHROMA_MC8_ALTIVEC_CORE
 140             }
 141         } else {
 142             vec_u8_t vsrcDuc;
 143             for (i = 0 ; i < h ; i++) {
 144                 vsrcCuc = vec_ld(stride + 0, src);
 145                 vsrcDuc = vec_ld(stride + 16, src);
 146                 vsrc2uc = vec_perm(vsrcCuc, vsrcDuc, vsrcperm0);
 147                 if (reallyBadAlign)
 148                     vsrc3uc = vsrcDuc;
 149                 else
 150                     vsrc3uc = vec_perm(vsrcCuc, vsrcDuc, vsrcperm1);
 151
 152                 CHROMA_MC8_ALTIVEC_CORE
 153             }
 154         }
 155     } else {
 156         const vec_s16_t vE = vec_add(vB, vC);
 157         if (ABCD[2]) { // x == 0 B == 0
 158             if (!loadSecond) {// -> !reallyBadAlign
 159                 for (i = 0 ; i < h ; i++) {
 160                     vsrcCuc = vec_ld(stride + 0, src);
 161                     vsrc1uc = vec_perm(vsrcCuc, vsrcCuc, vsrcperm0);
 162                     CHROMA_MC8_ALTIVEC_CORE_SIMPLE
 163
 164                     vsrc0uc = vsrc1uc;
 165                 }
 166             } else {
 167                 vec_u8_t vsrcDuc;
 168                 for (i = 0 ; i < h ; i++) {
 169                     vsrcCuc = vec_ld(stride + 0, src);
 170                     vsrcDuc = vec_ld(stride + 15, src);
 171                     vsrc1uc = vec_perm(vsrcCuc, vsrcDuc, vsrcperm0);
 172                     CHROMA_MC8_ALTIVEC_CORE_SIMPLE
 173
 174                     vsrc0uc = vsrc1uc;
 175                 }
 176             }
 177         } else { // y == 0 C == 0
 178             if (!loadSecond) {// -> !reallyBadAlign
 179                 for (i = 0 ; i < h ; i++) {
 180                     vsrcCuc = vec_ld(0, src);
 181                     vsrc0uc = vec_perm(vsrcCuc, vsrcCuc, vsrcperm0);
 182                     vsrc1uc = vec_perm(vsrcCuc, vsrcCuc, vsrcperm1);
 183
 184                     CHROMA_MC8_ALTIVEC_CORE_SIMPLE
 185                 }
 186             } else {
 187                 vec_u8_t vsrcDuc;
 188                 for (i = 0 ; i < h ; i++) {
 189                     vsrcCuc = vec_ld(0, src);
 190                     vsrcDuc = vec_ld(15, src);
 191                     vsrc0uc = vec_perm(vsrcCuc, vsrcDuc, vsrcperm0);
 192                     if (reallyBadAlign)
 193                         vsrc1uc = vsrcDuc;
 194                     else
 195                         vsrc1uc = vec_perm(vsrcCuc, vsrcDuc, vsrcperm1);
 196
 197                     CHROMA_MC8_ALTIVEC_CORE_SIMPLE
 198                 }
 199             }
 200         }
 201     }
 202     POWERPC_PERF_STOP_COUNT(PREFIX_h264_chroma_mc8_num, 1);
 203 }
 204
 205 #undef CHROMA_MC8_ALTIVEC_CORE
 206
 207 /* this code assume stride % 16 == 0 */
 208 static void PREFIX_h264_qpel16_h_lowpass_altivec(uint8_t * dst, uint8_t * src, int dstStride, int srcStride) {
 209     POWERPC_PERF_DECLARE(PREFIX_h264_qpel16_h_lowpass_num, 1);
 210     register int i;
 211
 212     LOAD_ZERO;
 213     const vec_u8_t permM2 = vec_lvsl(-2, src);
 214     const vec_u8_t permM1 = vec_lvsl(-1, src);
 215     const vec_u8_t permP0 = vec_lvsl(+0, src);
 216     const vec_u8_t permP1 = vec_lvsl(+1, src);
 217     const vec_u8_t permP2 = vec_lvsl(+2, src);
 218     const vec_u8_t permP3 = vec_lvsl(+3, src);
 219     const vec_s16_t v5ss = vec_splat_s16(5);
 220     const vec_u16_t v5us = vec_splat_u16(5);
 221     const vec_s16_t v20ss = vec_sl(vec_splat_s16(5),vec_splat_u16(2));
 222     const vec_s16_t v16ss = vec_sl(vec_splat_s16(1),vec_splat_u16(4));
 223
 224     vec_u8_t srcM2, srcM1, srcP0, srcP1, srcP2, srcP3;
 225
 226     register int align = ((((unsigned long)src) - 2) % 16);
 227
 228     vec_s16_t srcP0A, srcP0B, srcP1A, srcP1B,
 229               srcP2A, srcP2B, srcP3A, srcP3B,
 230               srcM1A, srcM1B, srcM2A, srcM2B,
 231               sum1A, sum1B, sum2A, sum2B, sum3A, sum3B,
 232               pp1A, pp1B, pp2A, pp2B, pp3A, pp3B,
 233               psumA, psumB, sumA, sumB;
 234
 235     vec_u8_t sum, vdst, fsum;
 236
 237     POWERPC_PERF_START_COUNT(PREFIX_h264_qpel16_h_lowpass_num, 1);
 238
 239     for (i = 0 ; i < 16 ; i ++) {
 240         vec_u8_t srcR1 = vec_ld(-2, src);
 241         vec_u8_t srcR2 = vec_ld(14, src);
 242
 243         switch (align) {
 244         default: {
 245             srcM2 = vec_perm(srcR1, srcR2, permM2);
 246             srcM1 = vec_perm(srcR1, srcR2, permM1);
 247             srcP0 = vec_perm(srcR1, srcR2, permP0);
 248             srcP1 = vec_perm(srcR1, srcR2, permP1);
 249             srcP2 = vec_perm(srcR1, srcR2, permP2);
 250             srcP3 = vec_perm(srcR1, srcR2, permP3);
 251         } break;
 252         case 11: {
 253             srcM2 = vec_perm(srcR1, srcR2, permM2);
 254             srcM1 = vec_perm(srcR1, srcR2, permM1);
 255             srcP0 = vec_perm(srcR1, srcR2, permP0);
 256             srcP1 = vec_perm(srcR1, srcR2, permP1);
 257             srcP2 = vec_perm(srcR1, srcR2, permP2);
 258             srcP3 = srcR2;
 259         } break;
 260         case 12: {
 261             vec_u8_t srcR3 = vec_ld(30, src);
 262             srcM2 = vec_perm(srcR1, srcR2, permM2);
 263             srcM1 = vec_perm(srcR1, srcR2, permM1);
 264             srcP0 = vec_perm(srcR1, srcR2, permP0);
 265             srcP1 = vec_perm(srcR1, srcR2, permP1);
 266             srcP2 = srcR2;
 267             srcP3 = vec_perm(srcR2, srcR3, permP3);
 268         } break;
 269         case 13: {
 270             vec_u8_t srcR3 = vec_ld(30, src);
 271             srcM2 = vec_perm(srcR1, srcR2, permM2);
 272             srcM1 = vec_perm(srcR1, srcR2, permM1);
 273             srcP0 = vec_perm(srcR1, srcR2, permP0);
 274             srcP1 = srcR2;
 275             srcP2 = vec_perm(srcR2, srcR3, permP2);
 276             srcP3 = vec_perm(srcR2, srcR3, permP3);
 277         } break;
 278         case 14: {
 279             vec_u8_t srcR3 = vec_ld(30, src);
 280             srcM2 = vec_perm(srcR1, srcR2, permM2);
 281             srcM1 = vec_perm(srcR1, srcR2, permM1);
 282             srcP0 = srcR2;
 283             srcP1 = vec_perm(srcR2, srcR3, permP1);
 284             srcP2 = vec_perm(srcR2, srcR3, permP2);
 285             srcP3 = vec_perm(srcR2, srcR3, permP3);
 286         } break;
 287         case 15: {
 288             vec_u8_t srcR3 = vec_ld(30, src);
 289             srcM2 = vec_perm(srcR1, srcR2, permM2);
 290             srcM1 = srcR2;
 291             srcP0 = vec_perm(srcR2, srcR3, permP0);
 292             srcP1 = vec_perm(srcR2, srcR3, permP1);
 293             srcP2 = vec_perm(srcR2, srcR3, permP2);
 294             srcP3 = vec_perm(srcR2, srcR3, permP3);
 295         } break;
 296         }
 297
 298         srcP0A = (vec_s16_t) vec_mergeh(zero_u8v, srcP0);
 299         srcP0B = (vec_s16_t) vec_mergel(zero_u8v, srcP0);
 300         srcP1A = (vec_s16_t) vec_mergeh(zero_u8v, srcP1);
 301         srcP1B = (vec_s16_t) vec_mergel(zero_u8v, srcP1);
 302
 303         srcP2A = (vec_s16_t) vec_mergeh(zero_u8v, srcP2);
 304         srcP2B = (vec_s16_t) vec_mergel(zero_u8v, srcP2);
 305         srcP3A = (vec_s16_t) vec_mergeh(zero_u8v, srcP3);
 306         srcP3B = (vec_s16_t) vec_mergel(zero_u8v, srcP3);
 307
 308         srcM1A = (vec_s16_t) vec_mergeh(zero_u8v, srcM1);
 309         srcM1B = (vec_s16_t) vec_mergel(zero_u8v, srcM1);
 310         srcM2A = (vec_s16_t) vec_mergeh(zero_u8v, srcM2);
 311         srcM2B = (vec_s16_t) vec_mergel(zero_u8v, srcM2);
 312
 313         sum1A = vec_adds(srcP0A, srcP1A);
 314         sum1B = vec_adds(srcP0B, srcP1B);
 315         sum2A = vec_adds(srcM1A, srcP2A);
 316         sum2B = vec_adds(srcM1B, srcP2B);
 317         sum3A = vec_adds(srcM2A, srcP3A);
 318         sum3B = vec_adds(srcM2B, srcP3B);
 319
 320         pp1A = vec_mladd(sum1A, v20ss, v16ss);
 321         pp1B = vec_mladd(sum1B, v20ss, v16ss);
 322
 323         pp2A = vec_mladd(sum2A, v5ss, zero_s16v);
 324         pp2B = vec_mladd(sum2B, v5ss, zero_s16v);
 325
 326         pp3A = vec_add(sum3A, pp1A);
 327         pp3B = vec_add(sum3B, pp1B);
 328
 329         psumA = vec_sub(pp3A, pp2A);
 330         psumB = vec_sub(pp3B, pp2B);
 331
 332         sumA = vec_sra(psumA, v5us);
 333         sumB = vec_sra(psumB, v5us);
 334
 335         sum = vec_packsu(sumA, sumB);
 336
 337         ASSERT_ALIGNED(dst);
 338         vdst = vec_ld(0, dst);
 339
 340         OP_U8_ALTIVEC(fsum, sum, vdst);
 341
 342         vec_st(fsum, 0, dst);
 343
 344         src += srcStride;
 345         dst += dstStride;
 346     }
 347     POWERPC_PERF_STOP_COUNT(PREFIX_h264_qpel16_h_lowpass_num, 1);
 348 }
 349
 350 /* this code assume stride % 16 == 0 */
 351 static void PREFIX_h264_qpel16_v_lowpass_altivec(uint8_t * dst, uint8_t * src, int dstStride, int srcStride) {
 352     POWERPC_PERF_DECLARE(PREFIX_h264_qpel16_v_lowpass_num, 1);
 353
 354     register int i;
 355
 356     LOAD_ZERO;
 357     const vec_u8_t perm = vec_lvsl(0, src);
 358     const vec_s16_t v20ss = vec_sl(vec_splat_s16(5),vec_splat_u16(2));
 359     const vec_u16_t v5us = vec_splat_u16(5);
 360     const vec_s16_t v5ss = vec_splat_s16(5);
 361     const vec_s16_t v16ss = vec_sl(vec_splat_s16(1),vec_splat_u16(4));
 362
 363     uint8_t *srcbis = src - (srcStride * 2);
 364
 365     const vec_u8_t srcM2a = vec_ld(0, srcbis);
 366     const vec_u8_t srcM2b = vec_ld(16, srcbis);
 367     const vec_u8_t srcM2 = vec_perm(srcM2a, srcM2b, perm);
 368     //srcbis += srcStride;
 369     const vec_u8_t srcM1a = vec_ld(0, srcbis += srcStride);
 370     const vec_u8_t srcM1b = vec_ld(16, srcbis);
 371     const vec_u8_t srcM1 = vec_perm(srcM1a, srcM1b, perm);
 372     //srcbis += srcStride;
 373     const vec_u8_t srcP0a = vec_ld(0, srcbis += srcStride);
 374     const vec_u8_t srcP0b = vec_ld(16, srcbis);
 375     const vec_u8_t srcP0 = vec_perm(srcP0a, srcP0b, perm);
 376     //srcbis += srcStride;
 377     const vec_u8_t srcP1a = vec_ld(0, srcbis += srcStride);
 378     const vec_u8_t srcP1b = vec_ld(16, srcbis);
 379     const vec_u8_t srcP1 = vec_perm(srcP1a, srcP1b, perm);
 380     //srcbis += srcStride;
 381     const vec_u8_t srcP2a = vec_ld(0, srcbis += srcStride);
 382     const vec_u8_t srcP2b = vec_ld(16, srcbis);
 383     const vec_u8_t srcP2 = vec_perm(srcP2a, srcP2b, perm);
 384     //srcbis += srcStride;
 385
 386     vec_s16_t srcM2ssA = (vec_s16_t) vec_mergeh(zero_u8v, srcM2);
 387     vec_s16_t srcM2ssB = (vec_s16_t) vec_mergel(zero_u8v, srcM2);
 388     vec_s16_t srcM1ssA = (vec_s16_t) vec_mergeh(zero_u8v, srcM1);
 389     vec_s16_t srcM1ssB = (vec_s16_t) vec_mergel(zero_u8v, srcM1);
 390     vec_s16_t srcP0ssA = (vec_s16_t) vec_mergeh(zero_u8v, srcP0);
 391     vec_s16_t srcP0ssB = (vec_s16_t) vec_mergel(zero_u8v, srcP0);
 392     vec_s16_t srcP1ssA = (vec_s16_t) vec_mergeh(zero_u8v, srcP1);
 393     vec_s16_t srcP1ssB = (vec_s16_t) vec_mergel(zero_u8v, srcP1);
 394     vec_s16_t srcP2ssA = (vec_s16_t) vec_mergeh(zero_u8v, srcP2);
 395     vec_s16_t srcP2ssB = (vec_s16_t) vec_mergel(zero_u8v, srcP2);
 396
 397     vec_s16_t pp1A, pp1B, pp2A, pp2B, pp3A, pp3B,
 398               psumA, psumB, sumA, sumB,
 399               srcP3ssA, srcP3ssB,
 400               sum1A, sum1B, sum2A, sum2B, sum3A, sum3B;
 401
 402     vec_u8_t sum, vdst, fsum, srcP3a, srcP3b, srcP3;
 403
 404     POWERPC_PERF_START_COUNT(PREFIX_h264_qpel16_v_lowpass_num, 1);
 405
 406     for (i = 0 ; i < 16 ; i++) {
 407         srcP3a = vec_ld(0, srcbis += srcStride);
 408         srcP3b = vec_ld(16, srcbis);
 409         srcP3 = vec_perm(srcP3a, srcP3b, perm);
 410         srcP3ssA = (vec_s16_t) vec_mergeh(zero_u8v, srcP3);
 411         srcP3ssB = (vec_s16_t) vec_mergel(zero_u8v, srcP3);
 412         //srcbis += srcStride;
 413
 414         sum1A = vec_adds(srcP0ssA, srcP1ssA);
 415         sum1B = vec_adds(srcP0ssB, srcP1ssB);
 416         sum2A = vec_adds(srcM1ssA, srcP2ssA);
 417         sum2B = vec_adds(srcM1ssB, srcP2ssB);
 418         sum3A = vec_adds(srcM2ssA, srcP3ssA);
 419         sum3B = vec_adds(srcM2ssB, srcP3ssB);
 420
 421         srcM2ssA = srcM1ssA;
 422         srcM2ssB = srcM1ssB;
 423         srcM1ssA = srcP0ssA;
 424         srcM1ssB = srcP0ssB;
 425         srcP0ssA = srcP1ssA;
 426         srcP0ssB = srcP1ssB;
 427         srcP1ssA = srcP2ssA;
 428         srcP1ssB = srcP2ssB;
 429         srcP2ssA = srcP3ssA;
 430         srcP2ssB = srcP3ssB;
 431
 432         pp1A = vec_mladd(sum1A, v20ss, v16ss);
 433         pp1B = vec_mladd(sum1B, v20ss, v16ss);
 434
 435         pp2A = vec_mladd(sum2A, v5ss, zero_s16v);
 436         pp2B = vec_mladd(sum2B, v5ss, zero_s16v);
 437
 438         pp3A = vec_add(sum3A, pp1A);
 439         pp3B = vec_add(sum3B, pp1B);
 440
 441         psumA = vec_sub(pp3A, pp2A);
 442         psumB = vec_sub(pp3B, pp2B);
 443
 444         sumA = vec_sra(psumA, v5us);
 445         sumB = vec_sra(psumB, v5us);
 446
 447         sum = vec_packsu(sumA, sumB);
 448
 449         ASSERT_ALIGNED(dst);
 450         vdst = vec_ld(0, dst);
 451
 452         OP_U8_ALTIVEC(fsum, sum, vdst);
 453
 454         vec_st(fsum, 0, dst);
 455
 456         dst += dstStride;
 457     }
 458     POWERPC_PERF_STOP_COUNT(PREFIX_h264_qpel16_v_lowpass_num, 1);
 459 }
 460
 461 /* this code assume stride % 16 == 0 *and* tmp is properly aligned */
 462 static void PREFIX_h264_qpel16_hv_lowpass_altivec(uint8_t * dst, int16_t * tmp, uint8_t * src, int dstStride, int tmpStride, int srcStride) {
 463     POWERPC_PERF_DECLARE(PREFIX_h264_qpel16_hv_lowpass_num, 1);
 464     register int i;
 465     LOAD_ZERO;
 466     const vec_u8_t permM2 = vec_lvsl(-2, src);
 467     const vec_u8_t permM1 = vec_lvsl(-1, src);
 468     const vec_u8_t permP0 = vec_lvsl(+0, src);
 469     const vec_u8_t permP1 = vec_lvsl(+1, src);
 470     const vec_u8_t permP2 = vec_lvsl(+2, src);
 471     const vec_u8_t permP3 = vec_lvsl(+3, src);
 472     const vec_s16_t v20ss = vec_sl(vec_splat_s16(5),vec_splat_u16(2));
 473     const vec_u32_t v10ui = vec_splat_u32(10);
 474     const vec_s16_t v5ss = vec_splat_s16(5);
 475     const vec_s16_t v1ss = vec_splat_s16(1);
 476     const vec_s32_t v512si = vec_sl(vec_splat_s32(1),vec_splat_u32(9));
 477     const vec_u32_t v16ui = vec_sl(vec_splat_u32(1),vec_splat_u32(4));
 478
 479     register int align = ((((unsigned long)src) - 2) % 16);
 480
 481     vec_s16_t srcP0A, srcP0B, srcP1A, srcP1B,
 482               srcP2A, srcP2B, srcP3A, srcP3B,
 483               srcM1A, srcM1B, srcM2A, srcM2B,
 484               sum1A, sum1B, sum2A, sum2B, sum3A, sum3B,
 485               pp1A, pp1B, pp2A, pp2B, psumA, psumB;
 486
 487     const vec_u8_t mperm = (const vec_u8_t)
 488         {0x00, 0x08, 0x01, 0x09, 0x02, 0x0A, 0x03, 0x0B,
 489          0x04, 0x0C, 0x05, 0x0D, 0x06, 0x0E, 0x07, 0x0F};
 490     int16_t *tmpbis = tmp;
 491
 492     vec_s16_t tmpM1ssA, tmpM1ssB, tmpM2ssA, tmpM2ssB,
 493               tmpP0ssA, tmpP0ssB, tmpP1ssA, tmpP1ssB,
 494               tmpP2ssA, tmpP2ssB;
 495
 496     vec_s32_t pp1Ae, pp1Ao, pp1Be, pp1Bo, pp2Ae, pp2Ao, pp2Be, pp2Bo,
 497               pp3Ae, pp3Ao, pp3Be, pp3Bo, pp1cAe, pp1cAo, pp1cBe, pp1cBo,
 498               pp32Ae, pp32Ao, pp32Be, pp32Bo, sumAe, sumAo, sumBe, sumBo,
 499               ssumAe, ssumAo, ssumBe, ssumBo;
 500     vec_u8_t fsum, sumv, sum, vdst;
 501     vec_s16_t ssume, ssumo;
 502
 503     POWERPC_PERF_START_COUNT(PREFIX_h264_qpel16_hv_lowpass_num, 1);
 504     src -= (2 * srcStride);
 505     for (i = 0 ; i < 21 ; i ++) {
 506         vec_u8_t srcM2, srcM1, srcP0, srcP1, srcP2, srcP3;
 507         vec_u8_t srcR1 = vec_ld(-2, src);
 508         vec_u8_t srcR2 = vec_ld(14, src);
 509
 510         switch (align) {
 511         default: {
 512             srcM2 = vec_perm(srcR1, srcR2, permM2);
 513             srcM1 = vec_perm(srcR1, srcR2, permM1);
 514             srcP0 = vec_perm(srcR1, srcR2, permP0);
 515             srcP1 = vec_perm(srcR1, srcR2, permP1);
 516             srcP2 = vec_perm(srcR1, srcR2, permP2);
 517             srcP3 = vec_perm(srcR1, srcR2, permP3);
 518         } break;
 519         case 11: {
 520             srcM2 = vec_perm(srcR1, srcR2, permM2);
 521             srcM1 = vec_perm(srcR1, srcR2, permM1);
 522             srcP0 = vec_perm(srcR1, srcR2, permP0);
 523             srcP1 = vec_perm(srcR1, srcR2, permP1);
 524             srcP2 = vec_perm(srcR1, srcR2, permP2);
 525             srcP3 = srcR2;
 526         } break;
 527         case 12: {
 528             vec_u8_t srcR3 = vec_ld(30, src);
 529             srcM2 = vec_perm(srcR1, srcR2, permM2);
 530             srcM1 = vec_perm(srcR1, srcR2, permM1);
 531             srcP0 = vec_perm(srcR1, srcR2, permP0);
 532             srcP1 = vec_perm(srcR1, srcR2, permP1);
 533             srcP2 = srcR2;
 534             srcP3 = vec_perm(srcR2, srcR3, permP3);
 535         } break;
 536         case 13: {
 537             vec_u8_t srcR3 = vec_ld(30, src);
 538             srcM2 = vec_perm(srcR1, srcR2, permM2);
 539             srcM1 = vec_perm(srcR1, srcR2, permM1);
 540             srcP0 = vec_perm(srcR1, srcR2, permP0);
 541             srcP1 = srcR2;
 542             srcP2 = vec_perm(srcR2, srcR3, permP2);
 543             srcP3 = vec_perm(srcR2, srcR3, permP3);
 544         } break;
 545         case 14: {
 546             vec_u8_t srcR3 = vec_ld(30, src);
 547             srcM2 = vec_perm(srcR1, srcR2, permM2);
 548             srcM1 = vec_perm(srcR1, srcR2, permM1);
 549             srcP0 = srcR2;
 550             srcP1 = vec_perm(srcR2, srcR3, permP1);
 551             srcP2 = vec_perm(srcR2, srcR3, permP2);
 552             srcP3 = vec_perm(srcR2, srcR3, permP3);
 553         } break;
 554         case 15: {
 555             vec_u8_t srcR3 = vec_ld(30, src);
 556             srcM2 = vec_perm(srcR1, srcR2, permM2);
 557             srcM1 = srcR2;
 558             srcP0 = vec_perm(srcR2, srcR3, permP0);
 559             srcP1 = vec_perm(srcR2, srcR3, permP1);
 560             srcP2 = vec_perm(srcR2, srcR3, permP2);
 561             srcP3 = vec_perm(srcR2, srcR3, permP3);
 562         } break;
 563         }
 564
 565         srcP0A = (vec_s16_t) vec_mergeh(zero_u8v, srcP0);
 566         srcP0B = (vec_s16_t) vec_mergel(zero_u8v, srcP0);
 567         srcP1A = (vec_s16_t) vec_mergeh(zero_u8v, srcP1);
 568         srcP1B = (vec_s16_t) vec_mergel(zero_u8v, srcP1);
 569
 570         srcP2A = (vec_s16_t) vec_mergeh(zero_u8v, srcP2);
 571         srcP2B = (vec_s16_t) vec_mergel(zero_u8v, srcP2);
 572         srcP3A = (vec_s16_t) vec_mergeh(zero_u8v, srcP3);
 573         srcP3B = (vec_s16_t) vec_mergel(zero_u8v, srcP3);
 574
 575         srcM1A = (vec_s16_t) vec_mergeh(zero_u8v, srcM1);
 576         srcM1B = (vec_s16_t) vec_mergel(zero_u8v, srcM1);
 577         srcM2A = (vec_s16_t) vec_mergeh(zero_u8v, srcM2);
 578         srcM2B = (vec_s16_t) vec_mergel(zero_u8v, srcM2);
 579
 580         sum1A = vec_adds(srcP0A, srcP1A);
 581         sum1B = vec_adds(srcP0B, srcP1B);
 582         sum2A = vec_adds(srcM1A, srcP2A);
 583         sum2B = vec_adds(srcM1B, srcP2B);
 584         sum3A = vec_adds(srcM2A, srcP3A);
 585         sum3B = vec_adds(srcM2B, srcP3B);
 586
 587         pp1A = vec_mladd(sum1A, v20ss, sum3A);
 588         pp1B = vec_mladd(sum1B, v20ss, sum3B);
 589
 590         pp2A = vec_mladd(sum2A, v5ss, zero_s16v);
 591         pp2B = vec_mladd(sum2B, v5ss, zero_s16v);
 592
 593         psumA = vec_sub(pp1A, pp2A);
 594         psumB = vec_sub(pp1B, pp2B);
 595
 596         vec_st(psumA, 0, tmp);
 597         vec_st(psumB, 16, tmp);
 598
 599         src += srcStride;
 600         tmp += tmpStride; /* int16_t*, and stride is 16, so it's OK here */
 601     }
 602
 603     tmpM2ssA = vec_ld(0, tmpbis);
 604     tmpM2ssB = vec_ld(16, tmpbis);
 605     tmpbis += tmpStride;
 606     tmpM1ssA = vec_ld(0, tmpbis);
 607     tmpM1ssB = vec_ld(16, tmpbis);
 608     tmpbis += tmpStride;
 609     tmpP0ssA = vec_ld(0, tmpbis);
 610     tmpP0ssB = vec_ld(16, tmpbis);
 611     tmpbis += tmpStride;
 612     tmpP1ssA = vec_ld(0, tmpbis);
 613     tmpP1ssB = vec_ld(16, tmpbis);
 614     tmpbis += tmpStride;
 615     tmpP2ssA = vec_ld(0, tmpbis);
 616     tmpP2ssB = vec_ld(16, tmpbis);
 617     tmpbis += tmpStride;
 618
 619     for (i = 0 ; i < 16 ; i++) {
 620         const vec_s16_t tmpP3ssA = vec_ld(0, tmpbis);
 621         const vec_s16_t tmpP3ssB = vec_ld(16, tmpbis);
 622
 623         const vec_s16_t sum1A = vec_adds(tmpP0ssA, tmpP1ssA);
 624         const vec_s16_t sum1B = vec_adds(tmpP0ssB, tmpP1ssB);
 625         const vec_s16_t sum2A = vec_adds(tmpM1ssA, tmpP2ssA);
 626         const vec_s16_t sum2B = vec_adds(tmpM1ssB, tmpP2ssB);
 627         const vec_s16_t sum3A = vec_adds(tmpM2ssA, tmpP3ssA);
 628         const vec_s16_t sum3B = vec_adds(tmpM2ssB, tmpP3ssB);
 629
 630         tmpbis += tmpStride;
 631
 632         tmpM2ssA = tmpM1ssA;
 633         tmpM2ssB = tmpM1ssB;
 634         tmpM1ssA = tmpP0ssA;
 635         tmpM1ssB = tmpP0ssB;
 636         tmpP0ssA = tmpP1ssA;
 637         tmpP0ssB = tmpP1ssB;
 638         tmpP1ssA = tmpP2ssA;
 639         tmpP1ssB = tmpP2ssB;
 640         tmpP2ssA = tmpP3ssA;
 641         tmpP2ssB = tmpP3ssB;
 642
 643         pp1Ae = vec_mule(sum1A, v20ss);
 644         pp1Ao = vec_mulo(sum1A, v20ss);
 645         pp1Be = vec_mule(sum1B, v20ss);
 646         pp1Bo = vec_mulo(sum1B, v20ss);
 647
 648         pp2Ae = vec_mule(sum2A, v5ss);
 649         pp2Ao = vec_mulo(sum2A, v5ss);
 650         pp2Be = vec_mule(sum2B, v5ss);
 651         pp2Bo = vec_mulo(sum2B, v5ss);
 652
 653         pp3Ae = vec_sra((vec_s32_t)sum3A, v16ui);
 654         pp3Ao = vec_mulo(sum3A, v1ss);
 655         pp3Be = vec_sra((vec_s32_t)sum3B, v16ui);
 656         pp3Bo = vec_mulo(sum3B, v1ss);
 657
 658         pp1cAe = vec_add(pp1Ae, v512si);
 659         pp1cAo = vec_add(pp1Ao, v512si);
 660         pp1cBe = vec_add(pp1Be, v512si);
 661         pp1cBo = vec_add(pp1Bo, v512si);
 662
 663         pp32Ae = vec_sub(pp3Ae, pp2Ae);
 664         pp32Ao = vec_sub(pp3Ao, pp2Ao);
 665         pp32Be = vec_sub(pp3Be, pp2Be);
 666         pp32Bo = vec_sub(pp3Bo, pp2Bo);
 667
 668         sumAe = vec_add(pp1cAe, pp32Ae);
 669         sumAo = vec_add(pp1cAo, pp32Ao);
 670         sumBe = vec_add(pp1cBe, pp32Be);
 671         sumBo = vec_add(pp1cBo, pp32Bo);
 672
 673         ssumAe = vec_sra(sumAe, v10ui);
 674         ssumAo = vec_sra(sumAo, v10ui);
 675         ssumBe = vec_sra(sumBe, v10ui);
 676         ssumBo = vec_sra(sumBo, v10ui);
 677
 678         ssume = vec_packs(ssumAe, ssumBe);
 679         ssumo = vec_packs(ssumAo, ssumBo);
 680
 681         sumv = vec_packsu(ssume, ssumo);
 682         sum = vec_perm(sumv, sumv, mperm);
 683
 684         ASSERT_ALIGNED(dst);
 685         vdst = vec_ld(0, dst);
 686
 687         OP_U8_ALTIVEC(fsum, sum, vdst);
 688
 689         vec_st(fsum, 0, dst);
 690
 691         dst += dstStride;
 692     }
 693     POWERPC_PERF_STOP_COUNT(PREFIX_h264_qpel16_hv_lowpass_num, 1);
 694 }