libaf/af_scaletempo.c

   1 /*
   2  * scaletempo audio filter
   3  *
   4  * scale tempo while maintaining pitch
   5  * (WSOLA technique with cross correlation)
   6  * inspired by SoundTouch library by Olli Parviainen
   7  *
   8  * basic algorithm
   9  *   - produce 'stride' output samples per loop
  10  *   - consume stride*scale input samples per loop
  11  *
  12  * to produce smoother transitions between strides, blend next overlap
  13  * samples from last stride with correlated samples of current input
  14  *
  15  * Copyright (c) 2007 Robert Juliano
  16  *
  17  * This file is part of MPlayer.
  18  *
  19  * MPlayer is free software; you can redistribute it and/or modify
  20  * it under the terms of the GNU General Public License as published by
  21  * the Free Software Foundation; either version 2 of the License, or
  22  * (at your option) any later version.
  23  *
  24  * MPlayer is distributed in the hope that it will be useful,
  25  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  26  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  27  * GNU General Public License for more details.
  28  *
  29  * You should have received a copy of the GNU General Public License along
  30  * with MPlayer; if not, write to the Free Software Foundation, Inc.,
  31  * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
  32  */
  33
  34 #include <stdlib.h>
  35 #include <string.h>
  36 #include <limits.h>
  37 #include <assert.h>
  38
  39 #include "af.h"
  40 #include "libavutil/common.h"
  41 #include "subopt-helper.h"
  42
  43 // Data for specific instances of this filter
  44 typedef struct af_scaletempo_s
  45 {
  46   // stride
  47   float   scale;
  48   float   speed;
  49   float   frames_stride_scaled;
  50   float   frames_stride_error;
  51   int     bytes_per_frame;
  52   int     bytes_stride;
  53   float   bytes_stride_scaled;
  54   int     bytes_queue;
  55   int     bytes_queued;
  56   int     bytes_to_slide;
  57   int8_t* buf_queue;
  58   // overlap
  59   int     samples_overlap;
  60   int     samples_standing;
  61   int     bytes_overlap;
  62   int     bytes_standing;
  63   void*   buf_overlap;
  64   void*   table_blend;
  65   void    (*output_overlap)(struct af_scaletempo_s* s, void* out_buf, int bytes_off);
  66   // best overlap
  67   int     frames_search;
  68   int     num_channels;
  69   void*   buf_pre_corr;
  70   void*   table_window;
  71   int     (*best_overlap_offset)(struct af_scaletempo_s* s);
  72   // command line
  73   float   scale_nominal;
  74   float   ms_stride;
  75   float   percent_overlap;
  76   float   ms_search;
  77   short   speed_tempo;
  78   short   speed_pitch;
  79 } af_scaletempo_t;
  80
  81 static int fill_queue(struct af_instance_s* af, af_data_t* data, int offset)
  82 {
  83   af_scaletempo_t* s = af->setup;
  84   int bytes_in = data->len - offset;
  85   int offset_unchanged = offset;
  86
  87   if (s->bytes_to_slide > 0) {
  88     if (s->bytes_to_slide < s->bytes_queued) {
  89       int bytes_move = s->bytes_queued - s->bytes_to_slide;
  90       memmove(s->buf_queue,
  91               s->buf_queue + s->bytes_to_slide,
  92               bytes_move);
  93       s->bytes_to_slide = 0;
  94       s->bytes_queued = bytes_move;
  95     } else {
  96       int bytes_skip;
  97       s->bytes_to_slide -= s->bytes_queued;
  98       bytes_skip = FFMIN(s->bytes_to_slide, bytes_in);
  99       s->bytes_queued = 0;
 100       s->bytes_to_slide -= bytes_skip;
 101       offset += bytes_skip;
 102       bytes_in -= bytes_skip;
 103     }
 104   }
 105
 106   if (bytes_in > 0) {
 107     int bytes_copy = FFMIN(s->bytes_queue - s->bytes_queued, bytes_in);
 108     assert(bytes_copy >= 0);
 109     memcpy(s->buf_queue + s->bytes_queued,
 110            (int8_t*)data->audio + offset,
 111            bytes_copy);
 112     s->bytes_queued += bytes_copy;
 113     offset += bytes_copy;
 114   }
 115
 116   return offset - offset_unchanged;
 117 }
 118
 119 #define UNROLL_PADDING (4*4)
 120
 121 static int best_overlap_offset_float(af_scaletempo_t* s)
 122 {
 123   float *pw, *po, *ppc, *search_start;
 124   float best_corr = INT_MIN;
 125   int best_off = 0;
 126   int i, off;
 127
 128   pw  = s->table_window;
 129   po  = s->buf_overlap;
 130   po += s->num_channels;
 131   ppc = s->buf_pre_corr;
 132   for (i=s->num_channels; i<s->samples_overlap; i++) {
 133     *ppc++ = *pw++ * *po++;
 134   }
 135
 136   search_start = (float*)s->buf_queue + s->num_channels;
 137   for (off=0; off<s->frames_search; off++) {
 138     float corr = 0;
 139     float* ps = search_start;
 140     ppc = s->buf_pre_corr;
 141     for (i=s->num_channels; i<s->samples_overlap; i++) {
 142       corr += *ppc++ * *ps++;
 143     }
 144     if (corr > best_corr) {
 145       best_corr = corr;
 146       best_off  = off;
 147     }
 148     search_start += s->num_channels;
 149   }
 150
 151   return best_off * 4 * s->num_channels;
 152 }
 153
 154 static int best_overlap_offset_s16(af_scaletempo_t* s)
 155 {
 156   int32_t *pw, *ppc;
 157   int16_t *po, *search_start;
 158   int64_t best_corr = INT64_MIN;
 159   int best_off = 0;
 160   int off;
 161   long i;
 162
 163   pw  = s->table_window;
 164   po  = s->buf_overlap;
 165   po += s->num_channels;
 166   ppc = s->buf_pre_corr;
 167   for (i=s->num_channels; i<s->samples_overlap; i++) {
 168     *ppc++ = ( *pw++ * *po++ ) >> 15;
 169   }
 170
 171   search_start = (int16_t*)s->buf_queue + s->num_channels;
 172   for (off=0; off<s->frames_search; off++) {
 173     int64_t corr = 0;
 174     int16_t* ps = search_start;
 175     ppc = s->buf_pre_corr;
 176     ppc += s->samples_overlap - s->num_channels;
 177     ps  += s->samples_overlap - s->num_channels;
 178     i  = -(s->samples_overlap - s->num_channels);
 179     do {
 180       corr += ppc[i+0] * ps[i+0];
 181       corr += ppc[i+1] * ps[i+1];
 182       corr += ppc[i+2] * ps[i+2];
 183       corr += ppc[i+3] * ps[i+3];
 184       i += 4;
 185     } while (i < 0);
 186     if (corr > best_corr) {
 187       best_corr = corr;
 188       best_off  = off;
 189     }
 190     search_start += s->num_channels;
 191   }
 192
 193   return best_off * 2 * s->num_channels;
 194 }
 195
 196 static void output_overlap_float(af_scaletempo_t* s, void* buf_out,
 197                                   int bytes_off)
 198 {
 199   float* pout = buf_out;
 200   float* pb   = s->table_blend;
 201   float* po   = s->buf_overlap;
 202   float* pin  = (float*)(s->buf_queue + bytes_off);
 203   int i;
 204   for (i=0; i<s->samples_overlap; i++) {
 205     *pout++ = *po - *pb++ * ( *po - *pin++ ); po++;
 206   }
 207 }
 208 static void output_overlap_s16(af_scaletempo_t* s, void* buf_out,
 209                                int bytes_off)
 210 {
 211   int16_t* pout = buf_out;
 212   int32_t* pb   = s->table_blend;
 213   int16_t* po   = s->buf_overlap;
 214   int16_t* pin  = (int16_t*)(s->buf_queue + bytes_off);
 215   int i;
 216   for (i=0; i<s->samples_overlap; i++) {
 217     *pout++ = *po - ( ( *pb++ * ( *po - *pin++ ) ) >> 16 ); po++;
 218   }
 219 }
 220
 221 // Filter data through filter
 222 static af_data_t* play(struct af_instance_s* af, af_data_t* data)
 223 {
 224   af_scaletempo_t* s = af->setup;
 225   int offset_in;
 226   int max_bytes_out;
 227   int8_t* pout;
 228
 229   if (s->scale == 1.0) {
 230     af->delay = 0;
 231     return data;
 232   }
 233
 234   // RESIZE_LOCAL_BUFFER - can't use macro
 235   max_bytes_out = ((int)(data->len / s->bytes_stride_scaled) + 1) * s->bytes_stride;
 236   if (max_bytes_out > af->data->len) {
 237     mp_msg(MSGT_AFILTER, MSGL_V, "[libaf] Reallocating memory in module %s, "
 238           "old len = %i, new len = %i\n",af->info->name,af->data->len,max_bytes_out);
 239     af->data->audio = realloc(af->data->audio, max_bytes_out);
 240     if (!af->data->audio) {
 241       mp_msg(MSGT_AFILTER, MSGL_FATAL, "[libaf] Could not allocate memory\n");
 242       return NULL;
 243     }
 244     af->data->len = max_bytes_out;
 245   }
 246
 247   offset_in = fill_queue(af, data, 0);
 248   pout = af->data->audio;
 249   while (s->bytes_queued >= s->bytes_queue) {
 250     int ti;
 251     float tf;
 252     int bytes_off = 0;
 253
 254     // output stride
 255     if (s->output_overlap) {
 256       if (s->best_overlap_offset)
 257         bytes_off = s->best_overlap_offset(s);
 258       s->output_overlap(s, pout, bytes_off);
 259     }
 260     memcpy(pout + s->bytes_overlap,
 261            s->buf_queue + bytes_off + s->bytes_overlap,
 262            s->bytes_standing);
 263     pout += s->bytes_stride;
 264
 265     // input stride
 266     memcpy(s->buf_overlap,
 267            s->buf_queue + bytes_off + s->bytes_stride,
 268            s->bytes_overlap);
 269     tf = s->frames_stride_scaled + s->frames_stride_error;
 270     ti = (int)tf;
 271     s->frames_stride_error = tf - ti;
 272     s->bytes_to_slide = ti * s->bytes_per_frame;
 273
 274     offset_in += fill_queue(af, data, offset_in);
 275   }
 276
 277   // This filter can have a negative delay when scale > 1:
 278   // output corresponding to some length of input can be decided and written
 279   // after receiving only a part of that input.
 280   af->delay = s->bytes_queued - s->bytes_to_slide;
 281
 282   data->audio = af->data->audio;
 283   data->len   = pout - (int8_t *)af->data->audio;
 284   return data;
 285 }
 286
 287 // Initialization and runtime control
 288 static int control(struct af_instance_s* af, int cmd, void* arg)
 289 {
 290   af_scaletempo_t* s = af->setup;
 291   switch(cmd){
 292   case AF_CONTROL_REINIT:{
 293     af_data_t* data = (af_data_t*)arg;
 294     float srate = data->rate / 1000;
 295     int nch = data->nch;
 296     int bps;
 297     int use_int = 0;
 298     int frames_stride, frames_overlap;
 299     int i, j;
 300
 301     mp_msg(MSGT_AFILTER, MSGL_V,
 302            "[scaletempo] %.3f speed * %.3f scale_nominal = %.3f\n",
 303            s->speed, s->scale_nominal, s->scale);
 304
 305     if (s->scale == 1.0) {
 306       if (s->speed_tempo && s->speed_pitch)
 307         return AF_DETACH;
 308       memcpy(af->data, data, sizeof(af_data_t));
 309       af->delay = 0;
 310       af->mul = 1;
 311       return af_test_output(af, data);
 312     }
 313
 314     af->data->rate = data->rate;
 315     af->data->nch  = data->nch;
 316     if ( data->format == AF_FORMAT_S16_LE
 317          || data->format == AF_FORMAT_S16_BE ) {
 318       use_int = 1;
 319       af->data->format  = AF_FORMAT_S16_NE;
 320       af->data->bps     = bps = 2;
 321     } else {
 322       af->data->format = AF_FORMAT_FLOAT_NE;
 323       af->data->bps    = bps = 4;
 324     }
 325
 326     frames_stride           = srate * s->ms_stride;
 327     s->bytes_stride         = frames_stride * bps * nch;
 328     s->bytes_stride_scaled  = s->scale * s->bytes_stride;
 329     s->frames_stride_scaled = s->scale * frames_stride;
 330     s->frames_stride_error  = 0;
 331     af->mul = (double)s->bytes_stride / s->bytes_stride_scaled;
 332     af->delay = 0;
 333
 334     frames_overlap = frames_stride * s->percent_overlap;
 335     if (frames_overlap <= 0) {
 336       s->bytes_standing   = s->bytes_stride;
 337       s->samples_standing = s->bytes_standing / bps;
 338       s->output_overlap   = NULL;
 339       s->bytes_overlap    = 0;
 340     } else {
 341       s->samples_overlap  = frames_overlap * nch;
 342       s->bytes_overlap    = frames_overlap * nch * bps;
 343       s->bytes_standing   = s->bytes_stride - s->bytes_overlap;
 344       s->samples_standing = s->bytes_standing / bps;
 345       s->buf_overlap      = realloc(s->buf_overlap, s->bytes_overlap);
 346       s->table_blend      = realloc(s->table_blend, s->bytes_overlap * 4);
 347       if(!s->buf_overlap || !s->table_blend) {
 348         mp_msg(MSGT_AFILTER, MSGL_FATAL, "[scaletempo] Out of memory\n");
 349         return AF_ERROR;
 350       }
 351       memset(s->buf_overlap, 0, s->bytes_overlap);
 352       if (use_int) {
 353         int32_t* pb = s->table_blend;
 354         int64_t blend = 0;
 355         for (i=0; i<frames_overlap; i++) {
 356           int32_t v = blend / frames_overlap;
 357           for (j=0; j<nch; j++) {
 358             *pb++ = v;
 359           }
 360           blend += 65536;  // 2^16
 361         }
 362         s->output_overlap = output_overlap_s16;
 363       } else {
 364         float* pb = s->table_blend;
 365         for (i=0; i<frames_overlap; i++) {
 366           float v = i / (float)frames_overlap;
 367           for (j=0; j<nch; j++) {
 368             *pb++ = v;
 369           }
 370         }
 371         s->output_overlap = output_overlap_float;
 372       }
 373     }
 374
 375     s->frames_search = (frames_overlap > 1) ? srate * s->ms_search : 0;
 376     if (s->frames_search <= 0) {
 377       s->best_overlap_offset = NULL;
 378     } else {
 379       if (use_int) {
 380         int64_t t = frames_overlap;
 381         int32_t n = 8589934588LL / (t * t);  // 4 * (2^31 - 1) / t^2
 382         int32_t* pw;
 383         s->buf_pre_corr = realloc(s->buf_pre_corr, s->bytes_overlap * 2 + UNROLL_PADDING);
 384         s->table_window = realloc(s->table_window, s->bytes_overlap * 2 - nch * bps * 2);
 385         if(!s->buf_pre_corr || !s->table_window) {
 386           mp_msg(MSGT_AFILTER, MSGL_FATAL, "[scaletempo] Out of memory\n");
 387           return AF_ERROR;
 388         }
 389         memset((char *)s->buf_pre_corr + s->bytes_overlap * 2, 0, UNROLL_PADDING);
 390         pw = s->table_window;
 391         for (i=1; i<frames_overlap; i++) {
 392           int32_t v = ( i * (t - i) * n ) >> 15;
 393           for (j=0; j<nch; j++) {
 394             *pw++ = v;
 395           }
 396         }
 397         s->best_overlap_offset = best_overlap_offset_s16;
 398       } else {
 399         float* pw;
 400         s->buf_pre_corr = realloc(s->buf_pre_corr, s->bytes_overlap);
 401         s->table_window = realloc(s->table_window, s->bytes_overlap - nch * bps);
 402         if(!s->buf_pre_corr || !s->table_window) {
 403           mp_msg(MSGT_AFILTER, MSGL_FATAL, "[scaletempo] Out of memory\n");
 404           return AF_ERROR;
 405         }
 406         pw = s->table_window;
 407         for (i=1; i<frames_overlap; i++) {
 408           float v = i * (frames_overlap - i);
 409           for (j=0; j<nch; j++) {
 410             *pw++ = v;
 411           }
 412         }
 413         s->best_overlap_offset = best_overlap_offset_float;
 414       }
 415     }
 416
 417     s->bytes_per_frame = bps * nch;
 418     s->num_channels    = nch;
 419
 420     s->bytes_queue
 421       = (s->frames_search + frames_stride + frames_overlap) * bps * nch;
 422     s->buf_queue = realloc(s->buf_queue, s->bytes_queue + UNROLL_PADDING);
 423     if(!s->buf_queue) {
 424       mp_msg(MSGT_AFILTER, MSGL_FATAL, "[scaletempo] Out of memory\n");
 425       return AF_ERROR;
 426     }
 427
 428     s->bytes_queued = 0;
 429     s->bytes_to_slide = 0;
 430
 431     mp_msg (MSGT_AFILTER, MSGL_DBG2, "[scaletempo] "
 432             "%.2f stride_in, %i stride_out, %i standing, "
 433             "%i overlap, %i search, %i queue, %s mode\n",
 434             s->frames_stride_scaled,
 435             (int)(s->bytes_stride / nch / bps),
 436             (int)(s->bytes_standing / nch / bps),
 437             (int)(s->bytes_overlap / nch / bps),
 438             s->frames_search,
 439             (int)(s->bytes_queue / nch / bps),
 440             (use_int?"s16":"float"));
 441
 442     return af_test_output(af, (af_data_t*)arg);
 443   }
 444   case AF_CONTROL_PLAYBACK_SPEED | AF_CONTROL_SET:{
 445     if (s->speed_tempo) {
 446       if (s->speed_pitch) {
 447         break;
 448       }
 449       s->speed = *(float*)arg;
 450       s->scale = s->speed * s->scale_nominal;
 451     } else {
 452       if (s->speed_pitch) {
 453         s->speed = 1 / *(float*)arg;
 454         s->scale = s->speed * s->scale_nominal;
 455         break;
 456       }
 457     }
 458     return AF_OK;
 459   }
 460   case AF_CONTROL_SCALETEMPO_AMOUNT | AF_CONTROL_SET:{
 461     s->scale = *(float*)arg;
 462     s->scale = s->speed * s->scale_nominal;
 463     return AF_OK;
 464   }
 465   case AF_CONTROL_SCALETEMPO_AMOUNT | AF_CONTROL_GET:
 466     *(float*)arg = s->scale;
 467     return AF_OK;
 468   case AF_CONTROL_COMMAND_LINE:{
 469     strarg_t speed = {};
 470     opt_t subopts[] = {
 471       {"scale",   OPT_ARG_FLOAT, &s->scale_nominal, NULL},
 472       {"stride",  OPT_ARG_FLOAT, &s->ms_stride, NULL},
 473       {"overlap", OPT_ARG_FLOAT, &s->percent_overlap, NULL},
 474       {"search",  OPT_ARG_FLOAT, &s->ms_search, NULL},
 475       {"speed",   OPT_ARG_STR,   &speed, NULL},
 476       {NULL},
 477     };
 478     if (subopt_parse(arg, subopts) != 0) {
 479       return AF_ERROR;
 480     }
 481     if (s->scale_nominal <= 0) {
 482         mp_msg(MSGT_AFILTER, MSGL_ERR, "[scaletempo] %s: %s: scale > 0\n",
 483                mp_gtext("error parsing command line"),
 484                mp_gtext("value out of range"));
 485       return AF_ERROR;
 486     }
 487     if (s->ms_stride <= 0) {
 488         mp_msg(MSGT_AFILTER, MSGL_ERR, "[scaletempo] %s: %s: stride > 0\n",
 489                mp_gtext("error parsing command line"),
 490                mp_gtext("value out of range"));
 491       return AF_ERROR;
 492     }
 493     if (s->percent_overlap < 0 || s->percent_overlap > 1) {
 494         mp_msg(MSGT_AFILTER, MSGL_ERR,
 495                "[scaletempo] %s: %s: 0 <= overlap <= 1\n",
 496                mp_gtext("error parsing command line"),
 497                mp_gtext("value out of range"));
 498       return AF_ERROR;
 499     }
 500     if (s->ms_search < 0) {
 501         mp_msg(MSGT_AFILTER, MSGL_ERR, "[scaletempo] %s: %s: search >= 0\n",
 502                mp_gtext("error parsing command line"),
 503                mp_gtext("value out of range"));
 504       return AF_ERROR;
 505     }
 506     if (speed.len > 0) {
 507       if (strcmp(speed.str, "pitch") == 0) {
 508         s->speed_tempo = 0;
 509         s->speed_pitch = 1;
 510       } else if (strcmp(speed.str, "tempo") == 0) {
 511         s->speed_tempo = 1;
 512         s->speed_pitch = 0;
 513       } else if (strcmp(speed.str, "none") == 0) {
 514         s->speed_tempo = 0;
 515         s->speed_pitch = 0;
 516       } else if (strcmp(speed.str, "both") == 0) {
 517         s->speed_tempo = 1;
 518         s->speed_pitch = 1;
 519       } else {
 520         mp_msg(MSGT_AFILTER, MSGL_ERR,
 521                "[scaletempo] %s: %s: speed=[pitch|tempo|none|both]\n",
 522                mp_gtext("error parsing command line"),
 523                mp_gtext("value out of range"));
 524         return AF_ERROR;
 525       }
 526     }
 527     s->scale = s->speed * s->scale_nominal;
 528     mp_msg(MSGT_AFILTER, MSGL_DBG2, "[scaletempo] %6.3f scale, %6.2f stride, %6.2f overlap, %6.2f search, speed = %s\n", s->scale_nominal, s->ms_stride, s->percent_overlap, s->ms_search, (s->speed_tempo?(s->speed_pitch?"tempo and speed":"tempo"):(s->speed_pitch?"pitch":"none")));
 529     return AF_OK;
 530   }
 531   }
 532   return AF_UNKNOWN;
 533 }
 534
 535 // Deallocate memory
 536 static void uninit(struct af_instance_s* af)
 537 {
 538   af_scaletempo_t* s = af->setup;
 539   free(af->data->audio);
 540   free(af->data);
 541   free(s->buf_queue);
 542   free(s->buf_overlap);
 543   free(s->buf_pre_corr);
 544   free(s->table_blend);
 545   free(s->table_window);
 546   free(af->setup);
 547 }
 548
 549 // Allocate memory and set function pointers
 550 static int af_open(af_instance_t* af){
 551   af_scaletempo_t* s;
 552
 553   af->control   = control;
 554   af->uninit    = uninit;
 555   af->play      = play;
 556   af->mul       = 1;
 557   af->data      = calloc(1,sizeof(af_data_t));
 558   af->setup     = calloc(1,sizeof(af_scaletempo_t));
 559   if(af->data == NULL || af->setup == NULL)
 560     return AF_ERROR;
 561
 562   s = af->setup;
 563   s->scale = s->speed = s->scale_nominal = 1.0;
 564   s->speed_tempo = 1;
 565   s->speed_pitch = 0;
 566   s->ms_stride = 60;
 567   s->percent_overlap = .20;
 568   s->ms_search = 14;
 569
 570   return AF_OK;
 571 }
 572
 573 // Description of this filter
 574 af_info_t af_info_scaletempo = {
 575   "Scale audio tempo while maintaining pitch",
 576   "scaletempo",
 577   "Robert Juliano",
 578   "",
 579   AF_FLAGS_REENTRANT,
 580   af_open
 581 };