modules/visualization/glspectrum.c

   1 /*****************************************************************************
   2  * glspectrum.c: spectrum visualization module based on OpenGL
   3  *****************************************************************************
   4  * Copyright © 2009-2013 VLC authors and VideoLAN
   5  *
   6  * Authors: Adrien Maglo <magsoft@videolan.org>
   7  *
   8  * This program is free software; you can redistribute it and/or modify it
   9  * under the terms of the GNU Lesser General Public License as published by
  10  * the Free Software Foundation; either version 2.1 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16  * GNU Lesser General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU Lesser General Public License
  19  * along with this program; if not, write to the Free Software Foundation,
  20  * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  21  *****************************************************************************/
  22
  23 #ifdef HAVE_CONFIG_H
  24 # include "config.h"
  25 #endif
  26
  27 #include <assert.h>
  28
  29 #include <vlc_common.h>
  30 #include <vlc_plugin.h>
  31 #include <vlc_aout.h>
  32 #include <vlc_vout_window.h>
  33 #include <vlc_opengl.h>
  34 #include <vlc_filter.h>
  35 #include <vlc_queue.h>
  36 #include <vlc_rand.h>
  37
  38 #ifdef __APPLE__
  39 # include <OpenGL/gl.h>
  40 #else
  41 # include <GL/gl.h>
  42 #endif
  43
  44 #include <math.h>
  45
  46 #include "visual/fft.h"
  47 #include "visual/window.h"
  48
  49
  50 /*****************************************************************************
  51  * Module descriptor
  52  *****************************************************************************/
  53 static int Open(vlc_object_t *);
  54 static void Close(filter_t *);
  55
  56 #define WIDTH_TEXT N_("Video width")
  57 #define WIDTH_LONGTEXT N_("The width of the visualization window, in pixels.")
  58
  59 #define HEIGHT_TEXT N_("Video height")
  60 #define HEIGHT_LONGTEXT N_("The height of the visualization window, in pixels.")
  61
  62 vlc_module_begin()
  63     set_shortname(N_("glSpectrum"))
  64     set_description(N_("3D OpenGL spectrum visualization"))
  65     set_capability("visualization", 0)
  66     set_category(CAT_AUDIO)
  67     set_subcategory(SUBCAT_AUDIO_VISUAL)
  68
  69     add_integer("glspectrum-width", 400, WIDTH_TEXT, WIDTH_LONGTEXT, false)
  70     add_integer("glspectrum-height", 300, HEIGHT_TEXT, HEIGHT_LONGTEXT, false)
  71
  72     add_shortcut("glspectrum")
  73     set_callback(Open)
  74 vlc_module_end()
  75
  76
  77 /*****************************************************************************
  78  * Local prototypes
  79  *****************************************************************************/
  80 typedef struct
  81 {
  82     vlc_thread_t thread;
  83
  84     /* Audio data */
  85     vlc_queue_t queue;
  86     bool dead;
  87     unsigned i_channels;
  88     unsigned i_prev_nb_samples;
  89     int16_t *p_prev_s16_buff;
  90
  91     /* Opengl */
  92     vlc_gl_t *gl;
  93
  94     float f_rotationAngle;
  95     float f_rotationIncrement;
  96
  97     /* FFT window parameters */
  98     window_param wind_param;
  99 } filter_sys_t;
 100
 101
 102 static block_t *DoWork(filter_t *, block_t *);
 103 static void *Thread(void *);
 104
 105 #define SPECTRUM_WIDTH 4.f
 106 #define NB_BANDS 20
 107 #define ROTATION_INCREMENT .1f
 108 #define BAR_DECREMENT .075f
 109 #define ROTATION_MAX 20
 110
 111 const GLfloat lightZeroColor[] = {1.0f, 1.0f, 1.0f, 1.0f};
 112 const GLfloat lightZeroPosition[] = {0.0f, 3.0f, 10.0f, 0.0f};
 113
 114 static const struct vlc_filter_operations filter_ops = {
 115     .filter_audio = DoWork, .close = Close,
 116 };
 117
 118 /**
 119  * Open the module.
 120  * @param p_this: the filter object
 121  * @return VLC_SUCCESS or vlc error codes
 122  */
 123 static int Open(vlc_object_t * p_this)
 124 {
 125     filter_t *p_filter = (filter_t *)p_this;
 126     filter_sys_t *p_sys = vlc_obj_malloc(p_this, sizeof (*p_sys));
 127
 128     if (p_sys == NULL)
 129         return VLC_ENOMEM;
 130
 131     p_filter->p_sys = p_sys;
 132
 133     /* Create the object for the thread */
 134     p_sys->i_channels = aout_FormatNbChannels(&p_filter->fmt_in.audio);
 135     p_sys->i_prev_nb_samples = 0;
 136     p_sys->p_prev_s16_buff = NULL;
 137
 138     p_sys->f_rotationAngle = 0;
 139     p_sys->f_rotationIncrement = ROTATION_INCREMENT;
 140
 141     /* Fetch the FFT window parameters */
 142     window_get_param( VLC_OBJECT( p_filter ), &p_sys->wind_param );
 143
 144     /* Create the FIFO for the audio data. */
 145     vlc_queue_Init(&p_sys->queue, offsetof (block_t, p_next));
 146     p_sys->dead = false;
 147
 148     /* Create the openGL provider */
 149     vout_window_cfg_t cfg = {
 150         .width = var_InheritInteger(p_filter, "glspectrum-width"),
 151         .height = var_InheritInteger(p_filter, "glspectrum-height"),
 152     };
 153
 154     p_sys->gl = vlc_gl_surface_Create(p_this, &cfg, NULL);
 155     if (p_sys->gl == NULL)
 156         return VLC_EGENERIC;
 157
 158     /* Create the thread */
 159     if (vlc_clone(&p_sys->thread, Thread, p_filter,
 160                   VLC_THREAD_PRIORITY_VIDEO)) {
 161         vlc_gl_surface_Destroy(p_sys->gl);
 162         return VLC_ENOMEM;
 163     }
 164
 165     p_filter->fmt_in.audio.i_format = VLC_CODEC_FL32;
 166     p_filter->fmt_out.audio = p_filter->fmt_in.audio;
 167     p_filter->ops = &filter_ops;
 168
 169     return VLC_SUCCESS;
 170 }
 171
 172
 173 /**
 174  * Close the module.
 175  * @param p_this: the filter object
 176  */
 177 static void Close(filter_t *p_filter)
 178 {
 179     filter_sys_t *p_sys = p_filter->p_sys;
 180
 181     /* Terminate the thread. */
 182     vlc_queue_Kill(&p_sys->queue, &p_sys->dead);
 183     vlc_join(p_sys->thread, NULL);
 184
 185     /* Free the ressources */
 186     vlc_gl_surface_Destroy(p_sys->gl);
 187     free(p_sys->p_prev_s16_buff);
 188 }
 189
 190
 191 /**
 192  * Do the actual work with the new sample.
 193  * @param p_filter: filter object
 194  * @param p_in_buf: input buffer
 195  */
 196 static block_t *DoWork(filter_t *p_filter, block_t *p_in_buf)
 197 {
 198     filter_sys_t *p_sys = p_filter->p_sys;
 199
 200     vlc_queue_Enqueue(&p_sys->queue, block_Duplicate(p_in_buf));
 201     return p_in_buf;
 202 }
 203
 204
 205 /**
 206   * Init the OpenGL scene.
 207   **/
 208 static void initOpenGLScene(void)
 209 {
 210     glEnable(GL_CULL_FACE);
 211     glEnable(GL_DEPTH_TEST);
 212     glDepthMask(GL_TRUE);
 213
 214     glMatrixMode(GL_PROJECTION);
 215     glFrustum(-1.0f, 1.0f, -1.0f, 1.0f, 0.5f, 10.0f);
 216
 217     glMatrixMode(GL_MODELVIEW);
 218     glTranslatef(0.0, -2.0, -2.0);
 219
 220     // Init the light.
 221     glEnable(GL_LIGHTING);
 222
 223     glColorMaterial(GL_FRONT, GL_AMBIENT_AND_DIFFUSE);
 224     glEnable(GL_COLOR_MATERIAL);
 225
 226     glEnable(GL_LIGHT0);
 227     glLightfv(GL_LIGHT0, GL_DIFFUSE, lightZeroColor);
 228     glLightfv(GL_LIGHT0, GL_POSITION, lightZeroPosition);
 229
 230     glShadeModel(GL_SMOOTH);
 231
 232     glEnable(GL_BLEND);
 233     glBlendFunc(GL_SRC_ALPHA, GL_ONE_MINUS_SRC_ALPHA);
 234 }
 235
 236
 237 /**
 238  * Draw one bar of the Spectrum.
 239  */
 240 static void drawBar(void)
 241 {
 242     const float w = SPECTRUM_WIDTH / NB_BANDS - 0.05f;
 243
 244     const GLfloat vertexCoords[] = {
 245         0.f, 0.f, 0.f,     w, 0.f, 0.f,   0.f, 1.f, 0.f,
 246         0.f, 1.f, 0.f,     w, 0.f, 0.f,     w, 1.f, 0.f,
 247
 248         0.f, 0.f, -w,    0.f, 0.f, 0.f,   0.f, 1.f,  -w,
 249         0.f, 1.f, -w,    0.f, 0.f, 0.f,   0.f, 1.f, 0.f,
 250
 251           w, 0.f, 0.f,     w, 0.f,  -w,     w, 1.f, 0.f,
 252           w, 1.f, 0.f,     w, 0.f,  -w,     w, 1.f,  -w,
 253
 254           w, 0.f,  -w,   0.f, 0.f,  -w,   0.f, 1.f,  -w,
 255         0.f, 1.f,  -w,     w, 1.f,  -w,     w, 0.f,  -w,
 256
 257         0.f, 1.f, 0.f,     w, 1.f, 0.f,     w, 1.f,  -w,
 258         0.f, 1.f, 0.f,     w, 1.f,  -w,   0.f, 1.f,  -w,
 259     };
 260
 261     const GLfloat normals[] = {
 262         0.f, 0.f, 1.f,   0.f, 0.f, 1.f,   0.f, 0.f, 1.f,
 263         0.f, 0.f, 1.f,   0.f, 0.f, 1.f,   0.f, 0.f, 1.f,
 264
 265         -1.f, 0.f, 0.f,   -1.f, 0.f, 0.f,   -1.f, 0.f, 0.f,
 266         -1.f, 0.f, 0.f,   -1.f, 0.f, 0.f,   -1.f, 0.f, 0.f,
 267
 268         1.f, 0.f, 0.f,   1.f, 0.f, 0.f,   1.f, 0.f, 0.f,
 269         1.f, 0.f, 0.f,   1.f, 0.f, 0.f,   1.f, 0.f, 0.f,
 270
 271         0.f, 0.f, -1.f,   0.f, 0.f, -1.f,   0.f, 0.f, -1.f,
 272         0.f, 0.f, -1.f,   0.f, 0.f, -1.f,   0.f, 0.f, -1.f,
 273
 274         0.f, 1.f, 0.f,   0.f, 1.f, 0.f,   0.f, 1.f, 0.f,
 275         0.f, 1.f, 0.f,   0.f, 1.f, 0.f,   0.f, 1.f, 0.f,
 276     };
 277
 278     glVertexPointer(3, GL_FLOAT, 0, vertexCoords);
 279     glNormalPointer(GL_FLOAT, 0, normals);
 280     glDrawArrays(GL_TRIANGLES, 0, 6 * 5);
 281 }
 282
 283
 284 /**
 285  * Set the color of one bar of the spectrum.
 286  * @param f_height the height of the bar.
 287  */
 288 static void setBarColor(float f_height)
 289 {
 290     float r, b;
 291
 292 #define BAR_MAX_HEIGHT 4.2f
 293     r = -1.f + 2 / BAR_MAX_HEIGHT * f_height;
 294     b = 2.f - 2 / BAR_MAX_HEIGHT * f_height;
 295 #undef BAR_MAX_HEIGHT
 296
 297     /* Test the ranges. */
 298     r = r > 1.f ? 1.f : r;
 299     b = b > 1.f ? 1.f : b;
 300
 301     r = r < 0.f ? 0.f : r;
 302     b = b < 0.f ? 0.f : b;
 303
 304     /* Set the bar color. */
 305     glColor4f(r, 0.f, b, 1.f);
 306 }
 307
 308
 309 /**
 310  * Draw all the bars of the spectrum.
 311  * @param heights the heights of all the bars.
 312  */
 313 static void drawBars(float heights[])
 314 {
 315     glPushMatrix();
 316     glTranslatef(-2.f, 0.f, 0.f);
 317
 318     glEnableClientState(GL_VERTEX_ARRAY);
 319     glEnableClientState(GL_NORMAL_ARRAY);
 320
 321     float w = SPECTRUM_WIDTH / NB_BANDS;
 322     for (unsigned i = 0; i < NB_BANDS; ++i)
 323     {
 324         glPushMatrix();
 325         glScalef(1.f, heights[i], 1.f);
 326         setBarColor(heights[i]);
 327         drawBar();
 328         glPopMatrix();
 329
 330         glTranslatef(w, 0.f, 0.f);
 331     }
 332
 333     glDisableClientState(GL_VERTEX_ARRAY);
 334     glDisableClientState(GL_NORMAL_ARRAY);
 335
 336     glPopMatrix();
 337 }
 338
 339
 340 /**
 341  * Update thread which do the rendering
 342  * @param p_this: the p_thread object
 343  */
 344 static void *Thread( void *p_data )
 345 {
 346     filter_t  *p_filter = (filter_t*)p_data;
 347     filter_sys_t *p_sys = p_filter->p_sys;
 348     vlc_gl_t *gl = p_sys->gl;
 349     block_t *block;
 350
 351     if (vlc_gl_MakeCurrent(gl) != VLC_SUCCESS)
 352     {
 353         msg_Err(p_filter, "Can't attach gl context");
 354         return NULL;
 355     }
 356     initOpenGLScene();
 357     vlc_gl_ReleaseCurrent(gl);
 358
 359     float height[NB_BANDS] = {0};
 360
 361     while ((block = vlc_queue_DequeueKillable(&p_sys->queue, &p_sys->dead)))
 362     {
 363         unsigned win_width, win_height;
 364
 365         vlc_gl_MakeCurrent(gl);
 366         if (vlc_gl_surface_CheckSize(gl, &win_width, &win_height))
 367             glViewport(0, 0, win_width, win_height);
 368
 369         /* Horizontal scale for 20-band equalizer */
 370         const unsigned xscale[] = {0,1,2,3,4,5,6,7,8,11,15,20,27,
 371                                    36,47,62,82,107,141,184,255};
 372
 373         fft_state *p_state = NULL; /* internal FFT data */
 374         DEFINE_WIND_CONTEXT(wind_ctx); /* internal window data */
 375
 376         unsigned i, j;
 377         float p_output[FFT_BUFFER_SIZE];           /* Raw FFT Result  */
 378         int16_t p_buffer1[FFT_BUFFER_SIZE];        /* Buffer on which we perform
 379                                                       the FFT (first channel) */
 380         int16_t p_dest[FFT_BUFFER_SIZE];           /* Adapted FFT result */
 381         float *p_buffl = (float*)block->p_buffer;  /* Original buffer */
 382
 383         int16_t  *p_buffs;                         /* int16_t converted buffer */
 384         int16_t  *p_s16_buff;                      /* int16_t converted buffer */
 385
 386         if (!block->i_nb_samples) {
 387             msg_Err(p_filter, "no samples yet");
 388             goto release;
 389         }
 390
 391         /* Allocate the buffer only if the number of samples change */
 392         if (block->i_nb_samples != p_sys->i_prev_nb_samples)
 393         {
 394             free(p_sys->p_prev_s16_buff);
 395             p_sys->p_prev_s16_buff = malloc(block->i_nb_samples *
 396                                             p_sys->i_channels *
 397                                             sizeof(int16_t));
 398             if (!p_sys->p_prev_s16_buff)
 399                 goto release;
 400             p_sys->i_prev_nb_samples = block->i_nb_samples;
 401         }
 402         p_buffs = p_s16_buff = p_sys->p_prev_s16_buff;
 403
 404         /* Convert the buffer to int16_t
 405            Pasted from float32tos16.c */
 406         for (i = block->i_nb_samples * p_sys->i_channels; i--;)
 407         {
 408             union {float f; int32_t i;} u;
 409
 410             u.f = *p_buffl + 384.f;
 411             if (u.i > 0x43c07fff)
 412                 *p_buffs = 32767;
 413             else if (u.i < 0x43bf8000)
 414                 *p_buffs = -32768;
 415             else
 416                 *p_buffs = u.i - 0x43c00000;
 417
 418             p_buffl++; p_buffs++;
 419         }
 420         p_state = visual_fft_init();
 421         if (!p_state)
 422         {
 423             msg_Err(p_filter,"unable to initialize FFT transform");
 424             goto release;
 425         }
 426         if (!window_init(FFT_BUFFER_SIZE, &p_sys->wind_param, &wind_ctx))
 427         {
 428             msg_Err(p_filter,"unable to initialize FFT window");
 429             goto release;
 430         }
 431         p_buffs = p_s16_buff;
 432         for (i = 0 ; i < FFT_BUFFER_SIZE; i++)
 433         {
 434             p_output[i] = 0;
 435             p_buffer1[i] = *p_buffs;
 436
 437             p_buffs += p_sys->i_channels;
 438             if (p_buffs >= &p_s16_buff[block->i_nb_samples * p_sys->i_channels])
 439                 p_buffs = p_s16_buff;
 440         }
 441         window_scale_in_place (p_buffer1, &wind_ctx);
 442         fft_perform (p_buffer1, p_output, p_state);
 443
 444         for (i = 0; i< FFT_BUFFER_SIZE; ++i)
 445             p_dest[i] = p_output[i] *  (2 ^ 16)
 446                         / ((FFT_BUFFER_SIZE / 2 * 32768) ^ 2);
 447
 448         for (i = 0 ; i < NB_BANDS; i++)
 449         {
 450             /* Decrease the previous size of the bar. */
 451             height[i] -= BAR_DECREMENT;
 452             if (height[i] < 0)
 453                 height[i] = 0;
 454
 455             int y = 0;
 456             /* We search the maximum on one scale
 457                to determine the current size of the bar. */
 458             for (j = xscale[i]; j < xscale[i + 1]; j++)
 459             {
 460                 if (p_dest[j] > y)
 461                      y = p_dest[j];
 462             }
 463             /* Calculate the height of the bar */
 464             float new_height = y != 0 ? logf(y) * 0.4f : 0;
 465             height[i] = new_height > height[i]
 466                         ? new_height : height[i];
 467         }
 468
 469         /* Determine the camera rotation angle. */
 470         p_sys->f_rotationAngle += p_sys->f_rotationIncrement;
 471         if (p_sys->f_rotationAngle <= -ROTATION_MAX)
 472             p_sys->f_rotationIncrement = ROTATION_INCREMENT;
 473         else if (p_sys->f_rotationAngle >= ROTATION_MAX)
 474             p_sys->f_rotationIncrement = -ROTATION_INCREMENT;
 475
 476         /* Render the frame. */
 477         glClear(GL_COLOR_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
 478
 479         glPushMatrix();
 480             glRotatef(p_sys->f_rotationAngle, 0, 1, 0);
 481             drawBars(height);
 482         glPopMatrix();
 483
 484         /* Wait to swapp the frame on time. */
 485         vlc_tick_wait(block->i_pts + (block->i_length / 2));
 486         vlc_gl_Swap(gl);
 487
 488 release:
 489         window_close(&wind_ctx);
 490         fft_close(p_state);
 491         vlc_gl_ReleaseCurrent(gl);
 492         block_Release(block);
 493     }
 494
 495     return NULL;
 496 }