libhb/deblock.c

   1 /*
   2  Copyright (C) 2005 Michael Niedermayer <michaelni@gmx.at>
   3
   4  This program is free software; you can redistribute it and/or modify
   5  it under the terms of the GNU General Public License as published by
   6  the Free Software Foundation; either version 2 of the License, or
   7  (at your option) any later version.
   8
   9  This program is distributed in the hope that it will be useful,
  10  but WITHOUT ANY WARRANTY; without even the implied warranty of
  11  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  12  GNU General Public License for more details.
  13
  14  You should have received a copy of the GNU General Public License
  15  along with this program; if not, write to the Free Software
  16  Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA
  17 */
  18
  19 #include "hb.h"
  20 #include "hbffmpeg.h"
  21
  22 #define PP7_QP_DEFAULT    5
  23 #define PP7_MODE_DEFAULT  2
  24
  25 #define XMIN(a,b) ((a) < (b) ? (a) : (b))
  26 #define XMAX(a,b) ((a) > (b) ? (a) : (b))
  27
  28 typedef short DCTELEM;
  29
  30 //===========================================================================//
  31 static const uint8_t  __attribute__((aligned(8))) pp7_dither[8][8] =
  32 {
  33     {  0,  48,  12,  60,   3,  51,  15,  63, },
  34     { 32,  16,  44,  28,  35,  19,  47,  31, },
  35     {  8,  56,   4,  52,  11,  59,   7,  55, },
  36     { 40,  24,  36,  20,  43,  27,  39,  23, },
  37     {  2,  50,  14,  62,   1,  49,  13,  61, },
  38     { 34,  18,  46,  30,  33,  17,  45,  29, },
  39     { 10,  58,   6,  54,   9,  57,   5,  53, },
  40     { 42,  26,  38,  22,  41,  25,  37,  21, },
  41 };
  42
  43 struct hb_filter_private_s
  44 {
  45     int           pp7_qp;
  46     int           pp7_mode;
  47     int           pp7_mpeg2;
  48     int           pp7_temp_stride;
  49     uint8_t     * pp7_src;
  50 };
  51
  52 static int hb_deblock_init( hb_filter_object_t * filter,
  53                             hb_filter_init_t * init );
  54
  55 static int hb_deblock_work( hb_filter_object_t * filter,
  56                             hb_buffer_t ** buf_in,
  57                             hb_buffer_t ** buf_out );
  58
  59 static void hb_deblock_close( hb_filter_object_t * filter );
  60
  61 hb_filter_object_t hb_filter_deblock =
  62 {
  63     .id            = HB_FILTER_DEBLOCK,
  64     .enforce_order = 1,
  65     .name          = "Deblock (pp7)",
  66     .settings      = NULL,
  67     .init          = hb_deblock_init,
  68     .work          = hb_deblock_work,
  69     .close         = hb_deblock_close,
  70 };
  71
  72 static inline void pp7_dct_a( DCTELEM * dst, uint8_t * src, int stride )
  73 {
  74     int i;
  75
  76     for( i = 0; i < 4; i++ )
  77     {
  78         int s0 =  src[0*stride] + src[6*stride];
  79         int s1 =  src[1*stride] + src[5*stride];
  80         int s2 =  src[2*stride] + src[4*stride];
  81         int s3 =  src[3*stride];
  82         int s  =  s3+s3;
  83
  84         s3 = s  - s0;
  85         s0 = s  + s0;
  86         s  = s2 + s1;
  87         s2 = s2 - s1;
  88
  89         dst[0] =   s0 + s;
  90         dst[2] =   s0 - s;
  91         dst[1] = 2*s3 + s2;
  92         dst[3] =   s3 - s2*2;
  93
  94         src++;
  95         dst += 4;
  96     }
  97 }
  98
  99 static void pp7_dct_b( DCTELEM * dst, DCTELEM * src )
 100 {
 101     int i;
 102
 103     for( i = 0; i < 4; i++ )
 104     {
 105         int s0 = src[0*4] + src[6*4];
 106         int s1 = src[1*4] + src[5*4];
 107         int s2 = src[2*4] + src[4*4];
 108         int s3 = src[3*4];
 109         int s  = s3+s3;
 110
 111         s3 = s  - s0;
 112         s0 = s  + s0;
 113         s  = s2 + s1;
 114         s2 = s2 - s1;
 115
 116         dst[0*4] =   s0 + s;
 117         dst[2*4] =   s0 - s;
 118         dst[1*4] = 2*s3 + s2;
 119         dst[3*4] =   s3 - s2*2;
 120
 121         src++;
 122         dst++;
 123     }
 124 }
 125
 126 #define N   (1<<16)
 127 #define N0  4
 128 #define N1  5
 129 #define N2  10
 130 #define SN0 2
 131 #define SN1 2.2360679775
 132 #define SN2 3.16227766017
 133
 134 static const int pp7_factor[16] =
 135 {
 136     N/(N0*N0), N/(N0*N1), N/(N0*N0),N/(N0*N2),
 137     N/(N1*N0), N/(N1*N1), N/(N1*N0),N/(N1*N2),
 138     N/(N0*N0), N/(N0*N1), N/(N0*N0),N/(N0*N2),
 139     N/(N2*N0), N/(N2*N1), N/(N2*N0),N/(N2*N2),
 140 };
 141
 142 static int pp7_threshold[99][16];
 143
 144 static void pp7_init_threshold( void )
 145 {
 146     int qp, i;
 147     int bias = 0;
 148
 149     for( qp = 0; qp < 99; qp++ )
 150     {
 151         for( i = 0; i < 16; i++ )
 152         {
 153             pp7_threshold[qp][i] =
 154                 ((i&1)?SN2:SN0) * ((i&4)?SN2:SN0) *
 155                  XMAX(1,qp) * (1<<2) - 1 - bias;
 156         }
 157     }
 158 }
 159
 160 static int pp7_hard_threshold( DCTELEM * src, int qp )
 161 {
 162     int i;
 163     int a;
 164
 165     a = src[0] * pp7_factor[0];
 166     for( i = 1; i < 16; i++ )
 167     {
 168         unsigned int threshold1 = pp7_threshold[qp][i];
 169         unsigned int threshold2 = (threshold1<<1);
 170         int level= src[i];
 171         if( ((unsigned)(level+threshold1)) > threshold2 )
 172         {
 173             a += level * pp7_factor[i];
 174         }
 175     }
 176     return (a + (1<<11)) >> 12;
 177 }
 178
 179 static int pp7_medium_threshold( DCTELEM * src, int qp )
 180 {
 181     int i;
 182     int a;
 183
 184     a = src[0] * pp7_factor[0];
 185     for( i = 1; i < 16; i++ )
 186     {
 187         unsigned int threshold1 = pp7_threshold[qp][i];
 188         unsigned int threshold2 = (threshold1<<1);
 189         int level= src[i];
 190         if( ((unsigned)(level+threshold1)) > threshold2 )
 191         {
 192             if( ((unsigned)(level+2*threshold1)) > 2*threshold2 )
 193             {
 194                 a += level * pp7_factor[i];
 195             }
 196             else
 197             {
 198                 if( level>0 )
 199                 {
 200                     a += 2*(level - (int)threshold1) * pp7_factor[i];
 201                 }
 202                 else
 203                 {
 204                     a += 2*(level + (int)threshold1) * pp7_factor[i];
 205                 }
 206             }
 207         }
 208     }
 209     return (a + (1<<11)) >> 12;
 210 }
 211
 212 static int pp7_soft_threshold( DCTELEM * src, int qp )
 213 {
 214     int i;
 215     int a;
 216
 217     a = src[0] * pp7_factor[0];
 218     for( i = 1; i < 16; i++ )
 219     {
 220         unsigned int threshold1 = pp7_threshold[qp][i];
 221         unsigned int threshold2 = (threshold1<<1);
 222         int level= src[i];
 223         if( ((unsigned)(level+threshold1))>threshold2 )
 224         {
 225             if( level>0 )
 226             {
 227                 a += (level - (int)threshold1) * pp7_factor[i];
 228             }
 229             else
 230             {
 231                 a += (level + (int)threshold1) * pp7_factor[i];
 232             }
 233         }
 234     }
 235     return (a + (1<<11)) >> 12;
 236 }
 237
 238 static int ( * pp7_requantize )( DCTELEM * src, int qp ) = pp7_hard_threshold;
 239
 240 static void pp7_filter( hb_filter_private_t * pv,
 241                         uint8_t * dst,
 242                         uint8_t * src,
 243                         int width,
 244                         int height,
 245                         uint8_t * qp_store,
 246                         int qp_stride,
 247                         int is_luma)
 248 {
 249     int x, y;
 250
 251     const int  stride = is_luma ? pv->pp7_temp_stride : ((width+16+15)&(~15));
 252     uint8_t  * p_src  = pv->pp7_src + 8*stride;
 253     DCTELEM  * block  = (DCTELEM *)(pv->pp7_src);
 254     DCTELEM  * temp   = (DCTELEM *)(pv->pp7_src + 32);
 255
 256     if( !src || !dst )
 257     {
 258         return;
 259     }
 260
 261     for( y = 0; y < height; y++ )
 262     {
 263         int index = 8 + 8*stride + y*stride;
 264         memcpy( p_src + index, src + y*width, width );
 265
 266         for( x = 0; x < 8; x++ )
 267         {
 268             p_src[index         - x - 1] = p_src[index +         x    ];
 269             p_src[index + width + x    ] = p_src[index + width - x - 1];
 270         }
 271     }
 272
 273     for( y = 0; y < 8; y++ )
 274     {
 275         memcpy( p_src + (     7-y)*stride,
 276                 p_src + (     y+8)*stride, stride );
 277         memcpy( p_src + (height+8+y)*stride,
 278                 p_src + (height-y+7)*stride, stride );
 279     }
 280
 281     for( y = 0; y < height; y++ )
 282     {
 283         for( x = -8; x < 0; x += 4 )
 284         {
 285             const int index = x + y*stride + (8-3)*(1+stride) + 8;
 286             uint8_t * src   = p_src + index;
 287             DCTELEM * tp    = temp+4*x;
 288
 289             pp7_dct_a( tp+4*8, src, stride );
 290         }
 291
 292         for( x = 0; x < width; )
 293         {
 294             const int qps = 3 + is_luma;
 295             int end = XMIN(x+8, width);
 296
 297             int qp;
 298             if( pv->pp7_qp )
 299             {
 300                 qp = pv->pp7_qp;
 301             }
 302             else
 303             {
 304                 qp = qp_store[ (XMIN(x, width-1)>>qps) +
 305                                (XMIN(y, height-1)>>qps) * qp_stride ];
 306
 307                 if( pv->pp7_mpeg2 )
 308                 {
 309                     qp >>= 1;
 310                 }
 311             }
 312
 313             for( ; x < end; x++ )
 314             {
 315                 const int index = x + y*stride + (8-3)*(1+stride) + 8;
 316                 uint8_t * src   = p_src + index;
 317                 DCTELEM * tp    = temp+4*x;
 318                 int v;
 319
 320                 if( (x&3) == 0 )
 321                 {
 322                     pp7_dct_a( tp+4*8, src, stride );
 323                 }
 324
 325                 pp7_dct_b( block, tp );
 326
 327                 v = pp7_requantize( block, qp );
 328                 v = (v + pp7_dither[y&7][x&7]) >> 6;
 329                 if( (unsigned)v > 255 )
 330                 {
 331                     v = (-v) >> 31;
 332                 }
 333                 dst[x + y*width] = v;
 334             }
 335         }
 336     }
 337 }
 338
 339 static int hb_deblock_init( hb_filter_object_t * filter,
 340                             hb_filter_init_t * init )
 341 {
 342     filter->private_data = calloc( sizeof(struct hb_filter_private_s), 1 );
 343     hb_filter_private_t * pv = filter->private_data;
 344
 345     pv->pp7_qp    = PP7_QP_DEFAULT;
 346     pv->pp7_mode  = PP7_MODE_DEFAULT;
 347     pv->pp7_mpeg2 = 1; /*mpi->qscale_type;*/
 348
 349     if( filter->settings )
 350     {
 351         sscanf( filter->settings, "%d:%d", &pv->pp7_qp, &pv->pp7_mode );
 352     }
 353
 354     if( pv->pp7_qp < 0 )
 355     {
 356         pv->pp7_qp = 0;
 357     }
 358
 359     pp7_init_threshold();
 360
 361     switch( pv->pp7_mode )
 362     {
 363         case 0:
 364             pp7_requantize = pp7_hard_threshold;
 365             break;
 366         case 1:
 367             pp7_requantize = pp7_soft_threshold;
 368             break;
 369         case 2:
 370             pp7_requantize = pp7_medium_threshold;
 371             break;
 372     }
 373
 374     int h = (init->geometry.height + 16 + 15) & (~15);
 375
 376     pv->pp7_temp_stride = (init->geometry.width + 16 + 15) & (~15);
 377
 378     pv->pp7_src = (uint8_t*)malloc( pv->pp7_temp_stride*(h+8)*sizeof(uint8_t) );
 379
 380     return 0;
 381 }
 382
 383 static void hb_deblock_close( hb_filter_object_t * filter )
 384 {
 385     hb_filter_private_t * pv = filter->private_data;
 386
 387     if( !pv )
 388     {
 389         return;
 390     }
 391
 392     free( pv );
 393     filter->private_data = NULL;
 394 }
 395
 396 static int hb_deblock_work( hb_filter_object_t * filter,
 397                             hb_buffer_t ** buf_in,
 398                             hb_buffer_t ** buf_out )
 399 {
 400     hb_filter_private_t * pv = filter->private_data;
 401     hb_buffer_t * in = *buf_in, * out;
 402
 403     if (in->s.flags & HB_BUF_FLAG_EOF)
 404     {
 405         *buf_out = in;
 406         *buf_in = NULL;
 407         return HB_FILTER_DONE;
 408     }
 409
 410     if( /*TODO: mpi->qscale ||*/ pv->pp7_qp )
 411     {
 412         out = hb_video_buffer_init( in->f.width, in->f.height );
 413
 414         pp7_filter( pv,
 415                 out->plane[0].data,
 416                 in->plane[0].data,
 417                 in->plane[0].stride,
 418                 in->plane[0].height,
 419                 NULL, /* TODO: mpi->qscale*/
 420                 0,    /* TODO: mpi->qstride*/
 421                 1 );
 422
 423         pp7_filter( pv,
 424                 out->plane[1].data,
 425                 in->plane[1].data,
 426                 in->plane[1].stride,
 427                 in->plane[1].height,
 428                 NULL, /* TODO: mpi->qscale*/
 429                 0,    /* TODO: mpi->qstride*/
 430                 0 );
 431
 432         pp7_filter( pv,
 433                 out->plane[2].data,
 434                 in->plane[2].data,
 435                 in->plane[2].stride,
 436                 in->plane[2].height,
 437                 NULL, /* TODO: mpi->qscale*/
 438                 0,    /* TODO: mpi->qstride*/
 439                 0 );
 440
 441         out->s = in->s;
 442         hb_buffer_move_subs( out, in );
 443
 444         *buf_out = out;
 445     }
 446     else
 447     {
 448         *buf_in = NULL;
 449         *buf_out = in;
 450     }
 451
 452     return HB_FILTER_OK;
 453 }