modules/audio_filter/scaletempo.c

   1 /*****************************************************************************
   2  * scaletempo.c: Scale audio tempo while maintaining pitch
   3  *****************************************************************************
   4  * Copyright © 2008 the VideoLAN team
   5  * $Id$
   6  *
   7  * Authors: Rov Juvano <rovjuvano@users.sourceforge.net>
   8  *
   9  * This program is free software; you can redistribute it and/or modify
  10  * it under the terms of the GNU General Public License as published by
  11  * the Free Software Foundation; either version 2 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * along with this program; if not, write to the Free Software
  21  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  22  *****************************************************************************/
  23
  24 /*****************************************************************************
  25  * Preamble
  26  *****************************************************************************/
  27 #ifdef HAVE_CONFIG_H
  28 # include "config.h"
  29 #endif
  30
  31 #include <vlc_common.h>
  32 #include <vlc_plugin.h>
  33 #include <vlc_aout.h>
  34 #include <vlc_filter.h>
  35
  36 #include <string.h> /* for memset */
  37 #include <limits.h> /* form INT_MIN */
  38
  39 /*****************************************************************************
  40  * Module descriptor
  41  *****************************************************************************/
  42 static int  Open( vlc_object_t * );
  43 static void Close( vlc_object_t * );
  44 static block_t *DoWork( filter_t *, block_t * );
  45
  46 vlc_module_begin ()
  47     set_description( N_("Audio tempo scaler synched with rate") )
  48     set_shortname( N_("Scaletempo") )
  49     set_capability( "audio filter", 0 )
  50     set_category( CAT_AUDIO )
  51     set_subcategory( SUBCAT_AUDIO_AFILTER )
  52
  53     add_integer_with_range( "scaletempo-stride", 30, 1, 2000,
  54         N_("Stride Length"), N_("Length in milliseconds to output each stride"), true )
  55     add_float_with_range( "scaletempo-overlap", .20, 0.0, 1.0,
  56         N_("Overlap Length"), N_("Percentage of stride to overlap"), true )
  57     add_integer_with_range( "scaletempo-search", 14, 0, 200,
  58         N_("Search Length"), N_("Length in milliseconds to search for best overlap position"), true )
  59
  60     set_callbacks( Open, Close )
  61 vlc_module_end ()
  62
  63 /*
  64  * Scaletempo works by producing audio in constant sized chunks (a "stride") but
  65  * consuming chunks proportional to the playback rate.
  66  *
  67  * Scaletempo then smooths the output by blending the end of one stride with
  68  * the next ("overlap").
  69  *
  70  * Scaletempo smooths the overlap further by searching within the input buffer
  71  * for the best overlap position.  Scaletempo uses a statistical cross correlation
  72  * (roughly a dot-product).  Scaletempo consumes most of its CPU cycles here.
  73  *
  74  * NOTE:
  75  * sample: a single audio sample for one channel
  76  * frame: a single set of samples, one for each channel
  77  * VLC uses these terms differently
  78  */
  79 struct filter_sys_t
  80 {
  81     /* Filter static config */
  82     double    scale;
  83     /* parameters */
  84     unsigned  ms_stride;
  85     double    percent_overlap;
  86     unsigned  ms_search;
  87     /* audio format */
  88     unsigned  samples_per_frame;  /* AKA number of channels */
  89     unsigned  bytes_per_sample;
  90     unsigned  bytes_per_frame;
  91     unsigned  sample_rate;
  92     /* stride */
  93     double    frames_stride_scaled;
  94     double    frames_stride_error;
  95     unsigned  bytes_stride;
  96     double    bytes_stride_scaled;
  97     unsigned  bytes_queue_max;
  98     unsigned  bytes_queued;
  99     unsigned  bytes_to_slide;
 100     uint8_t  *buf_queue;
 101     /* overlap */
 102     unsigned  samples_overlap;
 103     unsigned  samples_standing;
 104     unsigned  bytes_overlap;
 105     unsigned  bytes_standing;
 106     void     *buf_overlap;
 107     void     *table_blend;
 108     void    (*output_overlap)( filter_t *p_filter, void *p_out_buf, unsigned bytes_off );
 109     /* best overlap */
 110     unsigned  frames_search;
 111     void     *buf_pre_corr;
 112     void     *table_window;
 113     unsigned(*best_overlap_offset)( filter_t *p_filter );
 114 };
 115
 116 /*****************************************************************************
 117  * best_overlap_offset: calculate best offset for overlap
 118  *****************************************************************************/
 119 static unsigned best_overlap_offset_float( filter_t *p_filter )
 120 {
 121     filter_sys_t *p = p_filter->p_sys;
 122     float *pw, *po, *ppc, *search_start;
 123     float best_corr = INT_MIN;
 124     unsigned best_off = 0;
 125     unsigned i, off;
 126
 127     pw  = p->table_window;
 128     po  = p->buf_overlap;
 129     po += p->samples_per_frame;
 130     ppc = p->buf_pre_corr;
 131     for( i = p->samples_per_frame; i < p->samples_overlap; i++ ) {
 132       *ppc++ = *pw++ * *po++;
 133     }
 134
 135     search_start = (float *)p->buf_queue + p->samples_per_frame;
 136     for( off = 0; off < p->frames_search; off++ ) {
 137       float corr = 0;
 138       float *ps = search_start;
 139       ppc = p->buf_pre_corr;
 140       for( i = p->samples_per_frame; i < p->samples_overlap; i++ ) {
 141         corr += *ppc++ * *ps++;
 142       }
 143       if( corr > best_corr ) {
 144         best_corr = corr;
 145         best_off  = off;
 146       }
 147       search_start += p->samples_per_frame;
 148     }
 149
 150     return best_off * p->bytes_per_frame;
 151 }
 152
 153 /*****************************************************************************
 154  * output_overlap: blend end of previous stride with beginning of current stride
 155  *****************************************************************************/
 156 static void output_overlap_float( filter_t        *p_filter,
 157                                   void            *buf_out,
 158                                   unsigned         bytes_off )
 159 {
 160     filter_sys_t *p = p_filter->p_sys;
 161     float *pout = buf_out;
 162     float *pb   = p->table_blend;
 163     float *po   = p->buf_overlap;
 164     float *pin  = (float *)( p->buf_queue + bytes_off );
 165     unsigned i;
 166     for( i = 0; i < p->samples_overlap; i++ ) {
 167         *pout++ = *po - *pb++ * ( *po - *pin++ ); po++;
 168     }
 169 }
 170
 171 /*****************************************************************************
 172  * fill_queue: fill p_sys->buf_queue as much possible, skipping samples as needed
 173  *****************************************************************************/
 174 static size_t fill_queue( filter_t      *p_filter,
 175                           uint8_t       *p_buffer,
 176                           size_t         i_buffer,
 177                           size_t         offset )
 178 {
 179     filter_sys_t *p = p_filter->p_sys;
 180     unsigned bytes_in = i_buffer - offset;
 181     size_t offset_unchanged = offset;
 182
 183     if( p->bytes_to_slide > 0 ) {
 184         if( p->bytes_to_slide < p->bytes_queued ) {
 185             unsigned bytes_in_move = p->bytes_queued - p->bytes_to_slide;
 186             memmove( p->buf_queue,
 187                      p->buf_queue + p->bytes_to_slide,
 188                      bytes_in_move );
 189             p->bytes_to_slide = 0;
 190             p->bytes_queued   = bytes_in_move;
 191         } else {
 192             unsigned bytes_in_skip;
 193             p->bytes_to_slide -= p->bytes_queued;
 194             bytes_in_skip      = __MIN( p->bytes_to_slide, bytes_in );
 195             p->bytes_queued    = 0;
 196             p->bytes_to_slide -= bytes_in_skip;
 197             offset            += bytes_in_skip;
 198             bytes_in          -= bytes_in_skip;
 199         }
 200     }
 201
 202     if( bytes_in > 0 ) {
 203         unsigned bytes_in_copy = __MIN( p->bytes_queue_max - p->bytes_queued, bytes_in );
 204         memcpy( p->buf_queue + p->bytes_queued,
 205                 p_buffer + offset,
 206                 bytes_in_copy );
 207         p->bytes_queued += bytes_in_copy;
 208         offset          += bytes_in_copy;
 209     }
 210
 211     return offset - offset_unchanged;
 212 }
 213
 214 /*****************************************************************************
 215  * transform_buffer: main filter loop
 216  *****************************************************************************/
 217 static size_t transform_buffer( filter_t        *p_filter,
 218                                 uint8_t         *p_buffer,
 219                                 size_t           i_buffer,
 220                                 uint8_t         *pout )
 221 {
 222     filter_sys_t *p = p_filter->p_sys;
 223
 224     size_t offset_in = fill_queue( p_filter, p_buffer, i_buffer, 0 );
 225     unsigned bytes_out = 0;
 226     while( p->bytes_queued >= p->bytes_queue_max ) {
 227         unsigned bytes_off = 0;
 228
 229         // output stride
 230         if( p->output_overlap ) {
 231             if( p->best_overlap_offset ) {
 232                 bytes_off = p->best_overlap_offset( p_filter );
 233             }
 234             p->output_overlap( p_filter, pout, bytes_off );
 235         }
 236         memcpy( pout + p->bytes_overlap,
 237                 p->buf_queue + bytes_off + p->bytes_overlap,
 238                 p->bytes_standing );
 239         pout += p->bytes_stride;
 240         bytes_out += p->bytes_stride;
 241
 242         // input stride
 243         memcpy( p->buf_overlap,
 244                 p->buf_queue + bytes_off + p->bytes_stride,
 245                 p->bytes_overlap );
 246         double frames_to_slide = p->frames_stride_scaled + p->frames_stride_error;
 247         unsigned frames_to_stride_whole = (int)frames_to_slide;
 248         p->bytes_to_slide       = frames_to_stride_whole * p->bytes_per_frame;
 249         p->frames_stride_error  = frames_to_slide - frames_to_stride_whole;
 250
 251         offset_in += fill_queue( p_filter, p_buffer, i_buffer, offset_in );
 252     }
 253
 254     return bytes_out;
 255 }
 256
 257 /*****************************************************************************
 258  * calculate_output_buffer_size
 259  *****************************************************************************/
 260 static size_t calculate_output_buffer_size( filter_t        *p_filter,
 261                                             size_t           bytes_in )
 262 {
 263     filter_sys_t *p = p_filter->p_sys;
 264     size_t bytes_out = 0;
 265     int bytes_to_out = bytes_in + p->bytes_queued - p->bytes_to_slide;
 266     if( bytes_to_out >= (int)p->bytes_queue_max ) {
 267       /* while (total_buffered - stride_length * n >= queue_max) n++ */
 268       bytes_out = p->bytes_stride * ( (unsigned)(
 269           ( bytes_to_out - p->bytes_queue_max + /* rounding protection */ p->bytes_per_frame )
 270           / p->bytes_stride_scaled ) + 1 );
 271     }
 272     return bytes_out;
 273 }
 274
 275 /*****************************************************************************
 276  * reinit_buffers: reinitializes buffers in p_filter->p_sys
 277  *****************************************************************************/
 278 static int reinit_buffers( filter_t *p_filter )
 279 {
 280     filter_sys_t *p = p_filter->p_sys;
 281     unsigned i,j;
 282
 283     unsigned frames_stride = p->ms_stride * p->sample_rate / 1000.0;
 284     p->bytes_stride = frames_stride * p->bytes_per_frame;
 285
 286     /* overlap */
 287     unsigned frames_overlap = frames_stride * p->percent_overlap;
 288     if( frames_overlap < 1 )
 289     { /* if no overlap */
 290         p->bytes_overlap    = 0;
 291         p->bytes_standing   = p->bytes_stride;
 292         p->samples_standing = p->bytes_standing / p->bytes_per_sample;
 293         p->output_overlap   = NULL;
 294     }
 295     else
 296     {
 297         unsigned prev_overlap   = p->bytes_overlap;
 298         p->bytes_overlap    = frames_overlap * p->bytes_per_frame;
 299         p->samples_overlap  = frames_overlap * p->samples_per_frame;
 300         p->bytes_standing   = p->bytes_stride - p->bytes_overlap;
 301         p->samples_standing = p->bytes_standing / p->bytes_per_sample;
 302         p->buf_overlap      = malloc( p->bytes_overlap );
 303         p->table_blend      = malloc( p->samples_overlap * 4 ); /* sizeof (int32|float) */
 304         if( !p->buf_overlap || !p->table_blend )
 305             return VLC_ENOMEM;
 306         if( p->bytes_overlap > prev_overlap )
 307             memset( (uint8_t *)p->buf_overlap + prev_overlap, 0, p->bytes_overlap - prev_overlap );
 308
 309         float *pb = p->table_blend;
 310         float t = (float)frames_overlap;
 311         for( i = 0; i<frames_overlap; i++ )
 312         {
 313             float v = i / t;
 314             for( j = 0; j < p->samples_per_frame; j++ )
 315                 *pb++ = v;
 316         }
 317         p->output_overlap = output_overlap_float;
 318     }
 319
 320     /* best overlap */
 321     p->frames_search = ( frames_overlap <= 1 ) ? 0 : p->ms_search * p->sample_rate / 1000.0;
 322     if( p->frames_search < 1 )
 323     { /* if no search */
 324         p->best_overlap_offset = NULL;
 325     }
 326     else
 327     {
 328         unsigned bytes_pre_corr = ( p->samples_overlap - p->samples_per_frame ) * 4; /* sizeof (int32|float) */
 329         p->buf_pre_corr = malloc( bytes_pre_corr );
 330         p->table_window = malloc( bytes_pre_corr );
 331         if( ! p->buf_pre_corr || ! p->table_window )
 332             return VLC_ENOMEM;
 333         float *pw = p->table_window;
 334         for( i = 1; i<frames_overlap; i++ )
 335         {
 336             float v = i * ( frames_overlap - i );
 337             for( j = 0; j < p->samples_per_frame; j++ )
 338                 *pw++ = v;
 339         }
 340         p->best_overlap_offset = best_overlap_offset_float;
 341     }
 342
 343     unsigned new_size = ( p->frames_search + frames_stride + frames_overlap ) * p->bytes_per_frame;
 344     if( p->bytes_queued > new_size )
 345     {
 346         if( p->bytes_to_slide > p->bytes_queued )
 347         {
 348           p->bytes_to_slide -= p->bytes_queued;
 349           p->bytes_queued    = 0;
 350         }
 351         else
 352         {
 353             unsigned new_queued = __MIN( p->bytes_queued - p->bytes_to_slide, new_size );
 354             memmove( p->buf_queue,
 355                      p->buf_queue + p->bytes_queued - new_queued,
 356                      new_queued );
 357             p->bytes_to_slide = 0;
 358             p->bytes_queued   = new_queued;
 359         }
 360     }
 361     p->bytes_queue_max = new_size;
 362     p->buf_queue = malloc( p->bytes_queue_max );
 363     if( ! p->buf_queue )
 364         return VLC_ENOMEM;
 365
 366     p->bytes_stride_scaled  = p->bytes_stride * p->scale;
 367     p->frames_stride_scaled = p->bytes_stride_scaled / p->bytes_per_frame;
 368
 369     msg_Dbg( VLC_OBJECT(p_filter),
 370              "%.3f scale, %.3f stride_in, %i stride_out, %i standing, %i overlap, %i search, %i queue, %s mode",
 371              p->scale,
 372              p->frames_stride_scaled,
 373              (int)( p->bytes_stride / p->bytes_per_frame ),
 374              (int)( p->bytes_standing / p->bytes_per_frame ),
 375              (int)( p->bytes_overlap / p->bytes_per_frame ),
 376              p->frames_search,
 377              (int)( p->bytes_queue_max / p->bytes_per_frame ),
 378              "fl32");
 379
 380     return VLC_SUCCESS;
 381 }
 382
 383 /*****************************************************************************
 384  * Open: initialize as "audio filter"
 385  *****************************************************************************/
 386 static int Open( vlc_object_t *p_this )
 387 {
 388     filter_t     *p_filter = (filter_t *)p_this;
 389     filter_sys_t *p_sys;
 390     bool b_fit = true;
 391
 392     if( p_filter->fmt_in.audio.i_format != VLC_CODEC_FL32 ||
 393         p_filter->fmt_out.audio.i_format != VLC_CODEC_FL32 )
 394     {
 395         b_fit = false;
 396         p_filter->fmt_in.audio.i_format = p_filter->fmt_out.audio.i_format = VLC_CODEC_FL32;
 397         msg_Warn( p_filter, "bad input or output format" );
 398     }
 399     if( ! AOUT_FMTS_SIMILAR( &p_filter->fmt_in.audio, &p_filter->fmt_out.audio ) )
 400     {
 401         b_fit = false;
 402         memcpy( &p_filter->fmt_out.audio, &p_filter->fmt_in.audio, sizeof(audio_sample_format_t) );
 403         msg_Warn( p_filter, "input and output formats are not similar" );
 404     }
 405
 406     if( ! b_fit )
 407         return VLC_EGENERIC;
 408
 409     /* Allocate structure */
 410     p_sys = p_filter->p_sys = malloc( sizeof(*p_sys) );
 411     if( ! p_sys )
 412         return VLC_ENOMEM;
 413
 414     p_filter->pf_audio_filter = DoWork;
 415
 416     p_sys->scale             = 1.0;
 417     p_sys->sample_rate       = p_filter->fmt_in.audio.i_rate;
 418     p_sys->samples_per_frame = aout_FormatNbChannels( &p_filter->fmt_in.audio );
 419     p_sys->bytes_per_sample  = 4;
 420     p_sys->bytes_per_frame   = p_sys->samples_per_frame * p_sys->bytes_per_sample;
 421
 422     msg_Dbg( p_this, "format: %5i rate, %i nch, %i bps, %s",
 423              p_sys->sample_rate,
 424              p_sys->samples_per_frame,
 425              p_sys->bytes_per_sample,
 426              "fl32" );
 427
 428     p_sys->ms_stride       = var_InheritInteger( p_this, "scaletempo-stride" );
 429     p_sys->percent_overlap = var_InheritFloat( p_this, "scaletempo-overlap" );
 430     p_sys->ms_search       = var_InheritInteger( p_this, "scaletempo-search" );
 431
 432     msg_Dbg( p_this, "params: %i stride, %.3f overlap, %i search",
 433              p_sys->ms_stride, p_sys->percent_overlap, p_sys->ms_search );
 434
 435     p_sys->buf_queue      = NULL;
 436     p_sys->buf_overlap    = NULL;
 437     p_sys->table_blend    = NULL;
 438     p_sys->buf_pre_corr   = NULL;
 439     p_sys->table_window   = NULL;
 440     p_sys->bytes_overlap  = 0;
 441     p_sys->bytes_queued   = 0;
 442     p_sys->bytes_to_slide = 0;
 443     p_sys->frames_stride_error = 0;
 444
 445     if( reinit_buffers( p_filter ) != VLC_SUCCESS )
 446     {
 447         Close( p_this );
 448         return VLC_EGENERIC;
 449     }
 450     return VLC_SUCCESS;
 451 }
 452
 453 static void Close( vlc_object_t *p_this )
 454 {
 455     filter_t *p_filter = (filter_t *)p_this;
 456     filter_sys_t *p_sys = p_filter->p_sys;
 457     free( p_sys->buf_queue );
 458     free( p_sys->buf_overlap );
 459     free( p_sys->table_blend );
 460     free( p_sys->buf_pre_corr );
 461     free( p_sys->table_window );
 462     free( p_sys );
 463 }
 464
 465 /*****************************************************************************
 466  * DoWork: filter wrapper for transform_buffer
 467  *****************************************************************************/
 468 static block_t *DoWork( filter_t * p_filter, block_t * p_in_buf )
 469 {
 470     filter_sys_t *p = p_filter->p_sys;
 471
 472     if( p_filter->fmt_in.audio.i_rate == p->sample_rate )
 473         return p_in_buf;
 474
 475     double scale = p_filter->fmt_in.audio.i_rate / (double)p->sample_rate;
 476     if( scale != p->scale ) {
 477       p->scale = scale;
 478       p->bytes_stride_scaled  = p->bytes_stride * p->scale;
 479       p->frames_stride_scaled = p->bytes_stride_scaled / p->bytes_per_frame;
 480       p->bytes_to_slide = 0;
 481       msg_Dbg( p_filter, "%.3f scale, %.3f stride_in, %i stride_out",
 482                p->scale,
 483                p->frames_stride_scaled,
 484                (int)( p->bytes_stride / p->bytes_per_frame ) );
 485     }
 486
 487     size_t i_outsize = calculate_output_buffer_size ( p_filter, p_in_buf->i_buffer );
 488     block_t *p_out_buf = filter_NewAudioBuffer( p_filter, i_outsize );
 489     if( p_out_buf == NULL )
 490         return NULL;
 491
 492     size_t bytes_out = transform_buffer( p_filter,
 493         p_in_buf->p_buffer, p_in_buf->i_buffer,
 494         p_out_buf->p_buffer );
 495
 496     p_out_buf->i_buffer     = bytes_out;
 497     p_out_buf->i_nb_samples = bytes_out / p->bytes_per_frame;
 498     p_out_buf->i_dts        = p_in_buf->i_dts;
 499     p_out_buf->i_pts        = p_in_buf->i_pts;
 500     p_out_buf->i_length     = p_in_buf->i_length;
 501
 502     block_Release( p_in_buf );
 503     return p_out_buf;
 504 }