modules/visualization/glspectrum.c

   1 /*****************************************************************************
   2  * glspectrum.c: spectrum visualization module based on OpenGL
   3  *****************************************************************************
   4  * Copyright © 2009-2013 VLC authors and VideoLAN
   5  *
   6  * Authors: Adrien Maglo <magsoft@videolan.org>
   7  *
   8  * This program is free software; you can redistribute it and/or modify it
   9  * under the terms of the GNU Lesser General Public License as published by
  10  * the Free Software Foundation; either version 2.1 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16  * GNU Lesser General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU Lesser General Public License
  19  * along with this program; if not, write to the Free Software Foundation,
  20  * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  21  *****************************************************************************/
  22
  23 #ifdef HAVE_CONFIG_H
  24 # include "config.h"
  25 #endif
  26
  27 #include <assert.h>
  28
  29 #include <vlc_common.h>
  30 #include <vlc_plugin.h>
  31 #include <vlc_aout.h>
  32 #include <vlc_vout.h>
  33 #include <vlc_vout_wrapper.h>
  34 #include <vlc_opengl.h>
  35 #include <vlc_filter.h>
  36 #include <vlc_rand.h>
  37
  38 #include <GL/gl.h>
  39
  40 #include <math.h>
  41
  42 #include "visual/fft.h"
  43 #include "visual/window.h"
  44
  45
  46 /*****************************************************************************
  47  * Module descriptor
  48  *****************************************************************************/
  49 static int Open(vlc_object_t *);
  50 static void Close(vlc_object_t *);
  51
  52 #define WIDTH_TEXT N_("Video width")
  53 #define WIDTH_LONGTEXT N_("The width of the visualization window, in pixels.")
  54
  55 #define HEIGHT_TEXT N_("Video height")
  56 #define HEIGHT_LONGTEXT N_("The height of the visualization window, in pixels.")
  57
  58 vlc_module_begin()
  59     set_shortname(N_("glSpectrum"))
  60     set_description(N_("3D OpenGL spectrum visualization"))
  61     set_capability("visualization", 0)
  62     set_category(CAT_AUDIO)
  63     set_subcategory(SUBCAT_AUDIO_VISUAL)
  64
  65     add_integer("glspectrum-width", 400, WIDTH_TEXT, WIDTH_LONGTEXT, false)
  66     add_integer("glspectrum-height", 300, HEIGHT_TEXT, HEIGHT_LONGTEXT, false)
  67
  68     add_shortcut("glspectrum")
  69     set_callbacks(Open, Close)
  70 vlc_module_end()
  71
  72
  73 /*****************************************************************************
  74  * Local prototypes
  75  *****************************************************************************/
  76 struct filter_sys_t
  77 {
  78     vlc_thread_t thread;
  79     vlc_sem_t    ready;
  80     bool         b_error;
  81
  82     /* Audio data */
  83     unsigned i_channels;
  84     block_fifo_t    *fifo;
  85     unsigned i_prev_nb_samples;
  86     int16_t *p_prev_s16_buff;
  87
  88     /* Opengl */
  89     vout_thread_t  *p_vout;
  90     vout_display_t *p_vd;
  91
  92     float f_rotationAngle;
  93     float f_rotationIncrement;
  94
  95     /* Window size */
  96     int i_width;
  97     int i_height;
  98
  99     /* FFT window parameters */
 100     window_param wind_param;
 101 };
 102
 103
 104 static block_t *DoWork(filter_t *, block_t *);
 105 static void *Thread(void *);
 106
 107 #define SPECTRUM_WIDTH 4.0
 108 #define NB_BANDS 20
 109 #define ROTATION_INCREMENT 0.1
 110 #define BAR_DECREMENT 0.075
 111 #define ROTATION_MAX 20
 112
 113 const GLfloat lightZeroColor[] = {1.0f, 1.0f, 1.0f, 1.0f};
 114 const GLfloat lightZeroPosition[] = {0.0f, 3.0f, 10.0f, 0.0f};
 115
 116 /**
 117  * Open the module.
 118  * @param p_this: the filter object
 119  * @return VLC_SUCCESS or vlc error codes
 120  */
 121 static int Open(vlc_object_t * p_this)
 122 {
 123     filter_t *p_filter = (filter_t *)p_this;
 124     filter_sys_t *p_sys;
 125
 126     p_sys = p_filter->p_sys = (filter_sys_t*)malloc(sizeof(*p_sys));
 127     if (p_sys == NULL)
 128         return VLC_ENOMEM;
 129
 130     /* Create the object for the thread */
 131     vlc_sem_init(&p_sys->ready, 0);
 132     p_sys->b_error = false;
 133     p_sys->i_width = var_InheritInteger(p_filter, "glspectrum-width");
 134     p_sys->i_height = var_InheritInteger(p_filter, "glspectrum-height");
 135     p_sys->i_channels = aout_FormatNbChannels(&p_filter->fmt_in.audio);
 136     p_sys->i_prev_nb_samples = 0;
 137     p_sys->p_prev_s16_buff = NULL;
 138
 139     p_sys->f_rotationAngle = 0;
 140     p_sys->f_rotationIncrement = ROTATION_INCREMENT;
 141
 142     /* Fetch the FFT window parameters */
 143     window_get_param( VLC_OBJECT( p_filter ), &p_sys->wind_param );
 144
 145     /* Create the FIFO for the audio data. */
 146     p_sys->fifo = block_FifoNew();
 147     if (p_sys->fifo == NULL)
 148         goto error;
 149
 150     /* Create the thread */
 151     if (vlc_clone(&p_sys->thread, Thread, p_filter,
 152                   VLC_THREAD_PRIORITY_VIDEO))
 153         goto error;
 154
 155     /* Wait for the displaying thread to be ready. */
 156     vlc_sem_wait(&p_sys->ready);
 157     if (p_sys->b_error)
 158     {
 159         vlc_join(p_sys->thread, NULL);
 160         goto error;
 161     }
 162
 163     p_filter->fmt_in.audio.i_format = VLC_CODEC_FL32;
 164     p_filter->fmt_out.audio = p_filter->fmt_in.audio;
 165     p_filter->pf_audio_filter = DoWork;
 166
 167     return VLC_SUCCESS;
 168
 169 error:
 170     vlc_sem_destroy(&p_sys->ready);
 171     free(p_sys);
 172     return VLC_EGENERIC;
 173 }
 174
 175
 176 /**
 177  * Close the module.
 178  * @param p_this: the filter object
 179  */
 180 static void Close(vlc_object_t *p_this)
 181 {
 182     filter_t *p_filter = (filter_t *)p_this;
 183     filter_sys_t *p_sys = p_filter->p_sys;
 184
 185     /* Terminate the thread. */
 186     vlc_cancel(p_sys->thread);
 187     vlc_join(p_sys->thread, NULL);
 188
 189     /* Free the ressources */
 190     vout_DeleteDisplay(p_sys->p_vd, NULL);
 191     vlc_object_release(p_sys->p_vout);
 192
 193     block_FifoRelease(p_sys->fifo);
 194     free(p_sys->p_prev_s16_buff);
 195
 196     vlc_sem_destroy(&p_sys->ready);
 197     free(p_sys);
 198 }
 199
 200
 201 /**
 202  * Do the actual work with the new sample.
 203  * @param p_filter: filter object
 204  * @param p_in_buf: input buffer
 205  */
 206 static block_t *DoWork(filter_t *p_filter, block_t *p_in_buf)
 207 {
 208     block_t *block = block_Duplicate(p_in_buf);
 209     if (likely(block != NULL))
 210         block_FifoPut(p_filter->p_sys->fifo, block);
 211     return p_in_buf;
 212 }
 213
 214
 215 /**
 216   * Init the OpenGL scene.
 217   **/
 218 static void initOpenGLScene(void)
 219 {
 220     glEnable(GL_CULL_FACE);
 221     glEnable(GL_DEPTH_TEST);
 222     glDepthMask(GL_TRUE);
 223
 224     glMatrixMode(GL_PROJECTION);
 225     glFrustum(-1.0f, 1.0f, -1.0f, 1.0f, 0.5f, 10.0f);
 226
 227     glMatrixMode(GL_MODELVIEW);
 228     glTranslatef(0.0, -2.0, -2.0);
 229
 230     // Init the light.
 231     glEnable(GL_LIGHTING);
 232
 233     glColorMaterial(GL_FRONT, GL_AMBIENT_AND_DIFFUSE);
 234     glEnable(GL_COLOR_MATERIAL);
 235
 236     glEnable(GL_LIGHT0);
 237     glLightfv(GL_LIGHT0, GL_DIFFUSE, lightZeroColor);
 238     glLightfv(GL_LIGHT0, GL_POSITION, lightZeroPosition);
 239
 240     glShadeModel(GL_SMOOTH);
 241
 242     glEnable(GL_BLEND);
 243     glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
 244 }
 245
 246
 247 /**
 248  * Draw one bar of the Spectrum.
 249  */
 250 static void drawBar(void)
 251 {
 252     const float w = SPECTRUM_WIDTH / NB_BANDS - 0.05;
 253
 254     const GLfloat vertexCoords[] = {
 255         0.0, 0.0, 0.0,   w, 0.0, 0.0,   0.0, 1.0, 0.0,
 256         0.0, 1.0, 0.0,   w, 0.0, 0.0,   w  , 1.0, 0.0,
 257
 258         0.0, 0.0, -w,    0.0, 0.0, 0.0,   0.0, 1.0, -w,
 259         0.0, 1.0, -w,    0.0, 0.0, 0.0,   0.0, 1.0, 0.0,
 260
 261         w, 0.0, 0.0,     w, 0.0, -w,   w, 1.0, 0.0,
 262         w, 1.0, 0.0,     w, 0.0, -w,   w, 1.0, -w,
 263
 264         w, 0.0, -w,      0.0, 0.0, -w,  0.0, 1.0, -w,
 265         0.0, 1.0, -w,    w, 1.0, -w,    w, 0.0, -w,
 266
 267         0.0, 1.0, 0.0,   w, 1.0, 0.0,   w, 1.0, -w,
 268         0.0, 1.0, 0.0,   w, 1.0, -w,    0.0, 1.0, -w,
 269     };
 270
 271     const GLfloat normals[] = {
 272         0.0, 0.0, 1.0,   0.0, 0.0, 1.0,   0.0, 0.0, 1.0,
 273         0.0, 0.0, 1.0,   0.0, 0.0, 1.0,   0.0, 0.0, 1.0,
 274
 275         -1.0, 0.0, 0.0,   -1.0, 0.0, 0.0,   -1.0, 0.0, 0.0,
 276         -1.0, 0.0, 0.0,   -1.0, 0.0, 0.0,   -1.0, 0.0, 0.0,
 277
 278         1.0, 0.0, 0.0,   1.0, 0.0, 0.0,   1.0, 0.0, 0.0,
 279         1.0, 0.0, 0.0,   1.0, 0.0, 0.0,   1.0, 0.0, 0.0,
 280
 281         0.0, 0.0, -1.0,   0.0, 0.0, -1.0,   0.0, 0.0, -1.0,
 282         0.0, 0.0, -1.0,   0.0, 0.0, -1.0,   0.0, 0.0, -1.0,
 283
 284         0.0, 1.0, 0.0,   0.0, 1.0, 0.0,   0.0, 1.0, 0.0,
 285         0.0, 1.0, 0.0,   0.0, 1.0, 0.0,   0.0, 1.0, 0.0,
 286     };
 287
 288     glVertexPointer(3, GL_FLOAT, 0, vertexCoords);
 289     glNormalPointer(GL_FLOAT, 0, normals);
 290     glDrawArrays(GL_TRIANGLES, 0, 6 * 5);
 291 }
 292
 293
 294 /**
 295  * Set the color of one bar of the spectrum.
 296  * @param f_height the height of the bar.
 297  */
 298 static void setBarColor(float f_height)
 299 {
 300     float r, b;
 301
 302 #define BAR_MAX_HEIGHT 4.2
 303     r = -1.0 + 2 / BAR_MAX_HEIGHT * f_height;
 304     b = 2.0 - 2 / BAR_MAX_HEIGHT * f_height;
 305 #undef BAR_MAX_HEIGHT
 306
 307     /* Test the ranges. */
 308     r = r > 1.0 ? 1.0 : r;
 309     b = b > 1.0 ? 1.0 : b;
 310
 311     r = r < 0.0 ? 0.0 : r;
 312     b = b < 0.0 ? 0.0 : b;
 313
 314     /* Set the bar color. */
 315     glColor4f(r, 0.0, b, 1.0);
 316 }
 317
 318
 319 /**
 320  * Draw all the bars of the spectrum.
 321  * @param heights the heights of all the bars.
 322  */
 323 static void drawBars(float heights[])
 324 {
 325     glPushMatrix();
 326     glTranslatef(-2.0, 0.0, 0.0);
 327
 328     glEnableClientState(GL_VERTEX_ARRAY);
 329     glEnableClientState(GL_NORMAL_ARRAY);
 330
 331     float w = SPECTRUM_WIDTH / NB_BANDS;
 332     for (unsigned i = 0; i < NB_BANDS; ++i)
 333     {
 334         glPushMatrix();
 335         glScalef(1.0, heights[i], 1.0);
 336         setBarColor(heights[i]);
 337         drawBar();
 338         glPopMatrix();
 339
 340         glTranslatef(w, 0.0, 0.0);
 341     }
 342
 343     glDisableClientState(GL_VERTEX_ARRAY);
 344     glDisableClientState(GL_NORMAL_ARRAY);
 345
 346     glPopMatrix();
 347 }
 348
 349
 350 /**
 351  * Update thread which do the rendering
 352  * @param p_this: the p_thread object
 353  */
 354 static void *Thread( void *p_data )
 355 {
 356     filter_t  *p_filter = (filter_t*)p_data;
 357     filter_sys_t *p_sys = p_filter->p_sys;
 358
 359     video_format_t fmt;
 360     vlc_gl_t *gl;
 361     unsigned int i_last_width = 0;
 362     unsigned int i_last_height = 0;
 363
 364     /* Create the openGL provider */
 365     p_sys->p_vout =
 366         (vout_thread_t *)vlc_object_create(p_filter, sizeof(vout_thread_t));
 367     if (!p_sys->p_vout)
 368         goto error;
 369
 370     /* Configure the video format for the opengl provider. */
 371     video_format_Init(&fmt, 0);
 372     video_format_Setup(&fmt, VLC_CODEC_RGB32, p_sys->i_width, p_sys->i_height,
 373                        p_sys->i_width, p_sys->i_height, 0, 1 );
 374     fmt.i_sar_num = 1;
 375     fmt.i_sar_den = 1;
 376
 377     /* Init vout state. */
 378     vout_display_state_t state;
 379     memset(&state, 0, sizeof(state));
 380     state.cfg.display.sar.num = 1;
 381     state.cfg.display.sar.den = 1;
 382     state.cfg.is_display_filled = true;
 383     state.cfg.zoom.num = 1;
 384     state.cfg.zoom.den = 1;
 385     state.sar.num = 1;
 386     state.sar.den = 1;
 387
 388     p_sys->p_vd = vout_NewDisplay(p_sys->p_vout, &fmt, &state,
 389                                   "opengl", 1000000, 1000000);
 390     if (!p_sys->p_vd)
 391     {
 392         vlc_object_release(p_sys->p_vout);
 393         goto error;
 394     }
 395
 396     gl = vout_GetDisplayOpengl(p_sys->p_vd);
 397     if (!gl)
 398     {
 399         vout_DeleteDisplay(p_sys->p_vd, NULL);
 400         vlc_object_release(p_sys->p_vout);
 401         goto error;
 402     }
 403
 404     vlc_sem_post(&p_sys->ready);
 405
 406     vlc_gl_MakeCurrent(gl);
 407     initOpenGLScene();
 408     vlc_gl_ReleaseCurrent(gl);
 409
 410     float height[NB_BANDS] = {0};
 411
 412     while (1)
 413     {
 414         block_t *block = block_FifoGet(p_sys->fifo);
 415
 416         int canc = vlc_savecancel();
 417
 418         vlc_gl_MakeCurrent(gl);
 419         /* Manage the events */
 420         vout_ManageDisplay(p_sys->p_vd, true);
 421         if (p_sys->p_vd->cfg->display.width != i_last_width ||
 422             p_sys->p_vd->cfg->display.height != i_last_height)
 423         {
 424             /* FIXME it is not perfect as we will have black bands */
 425             vout_display_place_t place;
 426             vout_display_PlacePicture(&place, &p_sys->p_vd->source,
 427                                       p_sys->p_vd->cfg, false);
 428
 429             i_last_width  = p_sys->p_vd->cfg->display.width;
 430             i_last_height = p_sys->p_vd->cfg->display.height;
 431         }
 432
 433         /* Horizontal scale for 20-band equalizer */
 434         const unsigned xscale[] = {0,1,2,3,4,5,6,7,8,11,15,20,27,
 435                                    36,47,62,82,107,141,184,255};
 436
 437         fft_state *p_state = NULL; /* internal FFT data */
 438         DEFINE_WIND_CONTEXT(wind_ctx); /* internal window data */
 439
 440         unsigned i, j;
 441         float p_output[FFT_BUFFER_SIZE];           /* Raw FFT Result  */
 442         int16_t p_buffer1[FFT_BUFFER_SIZE];        /* Buffer on which we perform
 443                                                       the FFT (first channel) */
 444         int16_t p_dest[FFT_BUFFER_SIZE];           /* Adapted FFT result */
 445         float *p_buffl = (float*)block->p_buffer;  /* Original buffer */
 446
 447         int16_t  *p_buffs;                         /* int16_t converted buffer */
 448         int16_t  *p_s16_buff;                      /* int16_t converted buffer */
 449
 450         if (!block->i_nb_samples) {
 451             msg_Err(p_filter, "no samples yet");
 452             goto release;
 453         }
 454
 455         /* Allocate the buffer only if the number of samples change */
 456         if (block->i_nb_samples != p_sys->i_prev_nb_samples)
 457         {
 458             free(p_sys->p_prev_s16_buff);
 459             p_sys->p_prev_s16_buff = malloc(block->i_nb_samples *
 460                                             p_sys->i_channels *
 461                                             sizeof(int16_t));
 462             if (!p_sys->p_prev_s16_buff)
 463                 goto release;
 464             p_sys->i_prev_nb_samples = block->i_nb_samples;
 465         }
 466         p_buffs = p_s16_buff = p_sys->p_prev_s16_buff;
 467
 468         /* Convert the buffer to int16_t
 469            Pasted from float32tos16.c */
 470         for (i = block->i_nb_samples * p_sys->i_channels; i--;)
 471         {
 472             union {float f; int32_t i;} u;
 473
 474             u.f = *p_buffl + 384.0;
 475             if (u.i > 0x43c07fff)
 476                 *p_buffs = 32767;
 477             else if (u.i < 0x43bf8000)
 478                 *p_buffs = -32768;
 479             else
 480                 *p_buffs = u.i - 0x43c00000;
 481
 482             p_buffl++; p_buffs++;
 483         }
 484         p_state = visual_fft_init();
 485         if (!p_state)
 486         {
 487             msg_Err(p_filter,"unable to initialize FFT transform");
 488             goto release;
 489         }
 490         if (!window_init(FFT_BUFFER_SIZE, &p_sys->wind_param, &wind_ctx))
 491         {
 492             msg_Err(p_filter,"unable to initialize FFT window");
 493             goto release;
 494         }
 495         p_buffs = p_s16_buff;
 496         for (i = 0 ; i < FFT_BUFFER_SIZE; i++)
 497         {
 498             p_output[i] = 0;
 499             p_buffer1[i] = *p_buffs;
 500
 501             p_buffs += p_sys->i_channels;
 502             if (p_buffs >= &p_s16_buff[block->i_nb_samples * p_sys->i_channels])
 503                 p_buffs = p_s16_buff;
 504         }
 505         window_scale_in_place (p_buffer1, &wind_ctx);
 506         fft_perform (p_buffer1, p_output, p_state);
 507
 508         for (i = 0; i< FFT_BUFFER_SIZE; ++i)
 509             p_dest[i] = p_output[i] *  (2 ^ 16)
 510                         / ((FFT_BUFFER_SIZE / 2 * 32768) ^ 2);
 511
 512         for (i = 0 ; i < NB_BANDS; i++)
 513         {
 514             /* Decrease the previous size of the bar. */
 515             height[i] -= BAR_DECREMENT;
 516             if (height[i] < 0)
 517                 height[i] = 0;
 518
 519             int y = 0;
 520             /* We search the maximum on one scale
 521                to determine the current size of the bar. */
 522             for (j = xscale[i]; j < xscale[i + 1]; j++)
 523             {
 524                 if (p_dest[j] > y)
 525                      y = p_dest[j];
 526             }
 527             /* Calculate the height of the bar */
 528             float new_height = y != 0 ? log(y) * 0.4 : 0;
 529             height[i] = new_height > height[i]
 530                         ? new_height : height[i];
 531         }
 532
 533         /* Determine the camera rotation angle. */
 534         p_sys->f_rotationAngle += p_sys->f_rotationIncrement;
 535         if (p_sys->f_rotationAngle <= -ROTATION_MAX)
 536             p_sys->f_rotationIncrement = ROTATION_INCREMENT;
 537         else if (p_sys->f_rotationAngle >= ROTATION_MAX)
 538             p_sys->f_rotationIncrement = -ROTATION_INCREMENT;
 539
 540         /* Render the frame. */
 541         glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
 542
 543         glPushMatrix();
 544             glRotatef(p_sys->f_rotationAngle, 0, 1, 0);
 545             drawBars(height);
 546         glPopMatrix();
 547
 548         /* Wait to swapp the frame on time. */
 549         mwait(block->i_pts + (block->i_length / 2));
 550         if (!vlc_gl_Lock(gl))
 551         {
 552             vlc_gl_Swap(gl);
 553             vlc_gl_Unlock(gl);
 554         }
 555
 556 release:
 557         window_close(&wind_ctx);
 558         fft_close(p_state);
 559         vlc_gl_ReleaseCurrent(gl);
 560         block_Release(block);
 561         vlc_restorecancel(canc);
 562     }
 563
 564     assert(0);
 565
 566 error:
 567     p_sys->b_error = true;
 568     vlc_sem_post(&p_sys->ready);
 569     return NULL;
 570 }