modules/video_filter/opencv_wrapper.c

   1 /*****************************************************************************
   2  * opencv_wrapper.c : OpenCV wrapper video filter
   3  *****************************************************************************
   4  * Copyright (C) 2006-2012 VLC authors and VideoLAN
   5  * Copyright (C) 2012 Edward Wang
   6  *
   7  * Authors: Dugal Harris <dugalh@protoclea.co.za>
   8  *          Edward Wang <edward.c.wang@compdigitec.com>
   9  *
  10  * This program is free software; you can redistribute it and/or modify it
  11  * under the terms of the GNU Lesser General Public License as published by
  12  * the Free Software Foundation; either version 2.1 of the License, or
  13  * (at your option) any later version.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  18  * GNU Lesser General Public License for more details.
  19  *
  20  * You should have received a copy of the GNU Lesser General Public License
  21  * along with this program; if not, write to the Free Software Foundation,
  22  * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  23  *****************************************************************************/
  24
  25 /*****************************************************************************
  26  * Preamble
  27  *****************************************************************************/
  28
  29 #ifdef HAVE_CONFIG_H
  30 # include "config.h"
  31 #endif
  32
  33 #include <vlc_common.h>
  34 #include <vlc_plugin.h>
  35 #include <vlc_vout.h>
  36 #include <vlc_modules.h>
  37 #include <vlc_picture.h>
  38 #include <vlc_filter.h>
  39 #include <vlc_image.h>
  40 #include "filter_picture.h"
  41
  42 #include <opencv2/core/core_c.h>
  43 #include <opencv2/core/types_c.h>
  44
  45 /*****************************************************************************
  46  * Local prototypes
  47  *****************************************************************************/
  48 static int  Create    ( vlc_object_t * );
  49 static void Destroy   ( vlc_object_t * );
  50
  51 static picture_t* Filter( filter_t*, picture_t* );
  52
  53 static void ReleaseImages( filter_t* p_filter );
  54 static void VlcPictureToIplImage( filter_t* p_filter, picture_t* p_in );
  55
  56 /*****************************************************************************
  57  * Module descriptor
  58  *****************************************************************************/
  59
  60 static const char *const chroma_list[] = { "input", "I420", "RGB32"};
  61 static const char *const chroma_list_text[] = { N_("Use input chroma unaltered"),
  62   N_("I420 - first plane is grayscale"), N_("RGB32")};
  63
  64 static const char *const output_list[] = { "none", "input", "processed"};
  65 static const char *const output_list_text[] = { N_("Don't display any video"),
  66   N_("Display the input video"), N_("Display the processed video")};
  67
  68 static const char *const verbosity_list[] = { "error", "warning", "debug"};
  69 static const char *const verbosity_list_text[] = { N_("Show only errors"),
  70   N_("Show errors and warnings"), N_("Show everything including debug messages")};
  71
  72 vlc_module_begin ()
  73     set_description( N_("OpenCV video filter wrapper") )
  74     set_shortname( N_("OpenCV" ))
  75     set_category( CAT_VIDEO )
  76     set_subcategory( SUBCAT_VIDEO_VFILTER )
  77     set_capability( "video filter", 0 )
  78     add_shortcut( "opencv_wrapper" )
  79     set_callbacks( Create, Destroy )
  80     add_float_with_range( "opencv-scale", 1.0, 0.1, 2.0,
  81                           N_("Scale factor (0.1-2.0)"),
  82                           N_("Ammount by which to scale the picture before sending it to the internal OpenCV filter"),
  83                           false )
  84     add_string( "opencv-chroma", "input",
  85                           N_("OpenCV filter chroma"),
  86                           N_("Chroma to convert picture to before sending it to the internal OpenCV filter"), false);
  87         change_string_list( chroma_list, chroma_list_text )
  88     add_string( "opencv-output", "input",
  89                           N_("Wrapper filter output"),
  90                           N_("Determines what (if any) video is displayed by the wrapper filter"), false);
  91         change_string_list( output_list, output_list_text )
  92     add_string( "opencv-filter-name", "none",
  93                           N_("OpenCV internal filter name"),
  94                           N_("Name of internal OpenCV plugin filter to use"), false);
  95 vlc_module_end ()
  96
  97
  98 /*****************************************************************************
  99  * wrapper_output_t: what video is output
 100  *****************************************************************************/
 101 enum wrapper_output_t
 102 {
 103    NONE,
 104    VINPUT,
 105    PROCESSED
 106 };
 107
 108 /*****************************************************************************
 109  * internal_chroma_t: what chroma is sent to the internal opencv filter
 110  *****************************************************************************/
 111 enum internal_chroma_t
 112 {
 113    CINPUT,
 114    GREY,
 115    RGB
 116 };
 117
 118 /*****************************************************************************
 119  * filter_sys_t: opencv_wrapper video output method descriptor
 120  *****************************************************************************
 121  * This structure is part of the video output thread descriptor.
 122  * It describes the opencv_wrapper specific properties of an output thread.
 123  *****************************************************************************/
 124 struct filter_sys_t
 125 {
 126     image_handler_t *p_image;
 127
 128     int i_cv_image_size;
 129
 130     picture_t *p_proc_image;
 131     picture_t *p_to_be_freed;
 132
 133     float f_scale;
 134
 135     int i_wrapper_output;
 136     int i_internal_chroma;
 137
 138     IplImage *p_cv_image[VOUT_MAX_PLANES];
 139
 140     filter_t *p_opencv;
 141     char* psz_inner_name;
 142
 143     picture_t hacked_pic;
 144 };
 145
 146 /*****************************************************************************
 147  * Create: allocates opencv_wrapper video thread output method
 148  *****************************************************************************
 149  * This function allocates and initializes a opencv_wrapper vout method.
 150  *****************************************************************************/
 151 static int Create( vlc_object_t *p_this )
 152 {
 153     filter_t* p_filter = (filter_t*)p_this;
 154     char *psz_chroma, *psz_output;
 155
 156     /* Allocate structure */
 157     p_filter->p_sys = malloc( sizeof( filter_sys_t ) );
 158     if( p_filter->p_sys == NULL )
 159         return VLC_ENOMEM;
 160
 161     /* Load the internal OpenCV filter.
 162      *
 163      * This filter object is needed to call the internal OpenCV filter
 164      * for processing, the wrapper just converts into an IplImage* for
 165      * the other filter.
 166      *
 167      * We don't need to set up video formats for this filter as it not
 168      * actually using a picture_t.
 169      */
 170     p_filter->p_sys->p_opencv = vlc_object_create( p_filter, sizeof(filter_t) );
 171     if( !p_filter->p_sys->p_opencv ) {
 172         free( p_filter->p_sys );
 173         return VLC_ENOMEM;
 174     }
 175
 176     p_filter->p_sys->psz_inner_name = var_InheritString( p_filter, "opencv-filter-name" );
 177     if( p_filter->p_sys->psz_inner_name )
 178         p_filter->p_sys->p_opencv->p_module =
 179             module_need( p_filter->p_sys->p_opencv,
 180                          "opencv internal filter",
 181                          p_filter->p_sys->psz_inner_name,
 182                          true );
 183
 184     if( !p_filter->p_sys->p_opencv->p_module )
 185     {
 186         msg_Err( p_filter, "can't open internal opencv filter: %s", p_filter->p_sys->psz_inner_name );
 187         free( p_filter->p_sys->psz_inner_name );
 188         p_filter->p_sys->psz_inner_name = NULL;
 189         vlc_object_release( p_filter->p_sys->p_opencv );
 190         free( p_filter->p_sys );
 191
 192         return VLC_ENOMOD;
 193     }
 194
 195
 196     /* Init structure */
 197     p_filter->p_sys->p_image = image_HandlerCreate( p_filter );
 198     for( int i = 0; i < VOUT_MAX_PLANES; i++ )
 199         p_filter->p_sys->p_cv_image[i] = NULL;
 200     p_filter->p_sys->p_proc_image = NULL;
 201     p_filter->p_sys->p_to_be_freed = NULL;
 202     p_filter->p_sys->i_cv_image_size = 0;
 203
 204     /* Retrieve and apply config */
 205     psz_chroma = var_InheritString( p_filter, "opencv-chroma" );
 206     if( psz_chroma == NULL )
 207     {
 208         msg_Err( p_filter, "configuration variable %s empty, using 'grey'",
 209                          "opencv-chroma" );
 210         p_filter->p_sys->i_internal_chroma = GREY;
 211     } else if( !strcmp( psz_chroma, "input" ) )
 212         p_filter->p_sys->i_internal_chroma = CINPUT;
 213     else if( !strcmp( psz_chroma, "I420" ) )
 214         p_filter->p_sys->i_internal_chroma = GREY;
 215     else if( !strcmp( psz_chroma, "RGB32" ) )
 216         p_filter->p_sys->i_internal_chroma = RGB;
 217     else {
 218         msg_Err( p_filter, "no valid opencv-chroma provided, using 'grey'" );
 219         p_filter->p_sys->i_internal_chroma = GREY;
 220     }
 221
 222     free( psz_chroma );
 223
 224     psz_output = var_InheritString( p_filter, "opencv-output" );
 225     if( psz_output == NULL )
 226     {
 227         msg_Err( p_filter, "configuration variable %s empty, using 'input'",
 228                          "opencv-output" );
 229         p_filter->p_sys->i_wrapper_output = VINPUT;
 230     } else if( !strcmp( psz_output, "none" ) )
 231         p_filter->p_sys->i_wrapper_output = NONE;
 232     else if( !strcmp( psz_output, "input" ) )
 233         p_filter->p_sys->i_wrapper_output = VINPUT;
 234     else if( !strcmp( psz_output, "processed" ) )
 235         p_filter->p_sys->i_wrapper_output = PROCESSED;
 236     else {
 237         msg_Err( p_filter, "no valid opencv-output provided, using 'input'" );
 238         p_filter->p_sys->i_wrapper_output = VINPUT;
 239     }
 240     free( psz_output );
 241
 242     p_filter->p_sys->f_scale =
 243         var_InheritFloat( p_filter, "opencv-scale" );
 244
 245     msg_Info(p_filter, "Configuration: opencv-scale: %f, opencv-chroma: %d, "
 246         "opencv-output: %d, opencv-filter %s",
 247         p_filter->p_sys->f_scale,
 248         p_filter->p_sys->i_internal_chroma,
 249         p_filter->p_sys->i_wrapper_output,
 250         p_filter->p_sys->psz_inner_name);
 251
 252 #ifndef NDEBUG
 253     msg_Dbg( p_filter, "opencv_wrapper successfully started" );
 254 #endif
 255
 256     p_filter->pf_video_filter = Filter;
 257
 258     return VLC_SUCCESS;
 259 }
 260
 261 /*****************************************************************************
 262  * Destroy: destroy opencv_wrapper video thread output method
 263  *****************************************************************************
 264  * Terminate an output method created by opencv_wrapperCreateOutputMethod
 265  *****************************************************************************/
 266 static void Destroy( vlc_object_t *p_this )
 267 {
 268     filter_t* p_filter = (filter_t*)p_this;
 269     ReleaseImages( p_filter );
 270
 271     // Release the internal OpenCV filter.
 272     module_unneed( p_filter->p_sys->p_opencv, p_filter->p_sys->p_opencv->p_module );
 273     vlc_object_release( p_filter->p_sys->p_opencv );
 274     p_filter->p_sys->p_opencv = NULL;
 275
 276     free( p_filter->p_sys );
 277 }
 278
 279 /*****************************************************************************
 280  * ReleaseImages: Release OpenCV images in filter_sys_t.
 281  *****************************************************************************/
 282 static void ReleaseImages( filter_t* p_filter )
 283 {
 284     filter_sys_t* p_sys = p_filter->p_sys;
 285
 286     for( int i = 0; i < VOUT_MAX_PLANES; i++ )
 287     {
 288         if (p_sys->p_cv_image[i] != NULL)
 289         {
 290             cvReleaseImageHeader(&(p_sys->p_cv_image[i]));
 291             p_sys->p_cv_image[i] = NULL;
 292         }
 293     }
 294     p_sys->i_cv_image_size = 0;
 295
 296     /* Release temp picture_t if it exists */
 297     if (p_sys->p_to_be_freed)
 298     {
 299         picture_Release( p_sys->p_to_be_freed );
 300         p_sys->p_to_be_freed = NULL;
 301     }
 302
 303 #ifndef NDEBUG
 304     msg_Dbg( p_filter, "images released" );
 305 #endif
 306 }
 307
 308 /*****************************************************************************
 309  * VlcPictureToIplImage: Convert picture_t to IplImage
 310  *****************************************************************************
 311  * Converts given picture_t into IplImage(s) according to module config.
 312  * IplImage(s) are stored in vout_sys_t.
 313  *****************************************************************************/
 314 static void VlcPictureToIplImage( filter_t* p_filter, picture_t* p_in )
 315 {
 316     int planes = p_in->i_planes;    //num input video planes
 317     // input video size
 318     CvSize sz = cvSize(abs(p_in->format.i_width), abs(p_in->format.i_height));
 319     video_format_t fmt_out;
 320     filter_sys_t* p_sys = p_filter->p_sys;
 321
 322     memset( &fmt_out, 0, sizeof(video_format_t) );
 323
 324     //do scale / color conversion according to p_sys config
 325     if ((p_sys->f_scale != 1) || (p_sys->i_internal_chroma != CINPUT))
 326     {
 327         fmt_out = p_in->format;
 328
 329         //calc the scaled video size
 330         fmt_out.i_width = p_in->format.i_width * p_sys->f_scale;
 331         fmt_out.i_height = p_in->format.i_height * p_sys->f_scale;
 332
 333         if (p_sys->i_internal_chroma == RGB)
 334         {
 335             //rgb2 gives 3 separate planes, this gives 1 interleaved plane
 336             //rv24 gives is about 20% faster but gives r&b the wrong way round
 337             //and I can't think of an easy way to fix this
 338             fmt_out.i_chroma = VLC_CODEC_RGB24;
 339         }
 340         else if (p_sys->i_internal_chroma == GREY)
 341         {
 342             //take the I (gray) plane (video seems to commonly be in this fmt so usually the
 343             //conversion does nothing)
 344             fmt_out.i_chroma = VLC_CODEC_I420;
 345         }
 346
 347         //convert from the input image
 348         p_sys->p_proc_image = image_Convert( p_sys->p_image, p_in,
 349                                      &(p_in->format), &fmt_out );
 350
 351         if (!p_sys->p_proc_image)
 352         {
 353             msg_Err(p_filter, "can't convert (unsupported formats?), aborting...");
 354             return;
 355         }
 356
 357         p_sys->p_to_be_freed = p_sys->p_proc_image;    //remember this so we can free it later
 358
 359     }
 360     else    //((p_sys->f_scale != 1) || (p_sys->i_internal_chroma != CINPUT))
 361     {
 362         // In theory, you could use the input image without conversion,
 363         // but it seems to cause weird picture effects (like repeated
 364         // image filtering) and picture leaking.
 365         p_sys->p_proc_image = filter_NewPicture( p_filter ); //p_in
 366         picture_Copy( p_sys->p_proc_image, p_in );
 367         p_sys->p_to_be_freed = p_sys->p_proc_image;
 368     }
 369
 370     //Convert to the IplImage array that is to be processed.
 371     //If there are multiple planes in p_sys->p_proc_image, then 1 IplImage
 372     //is created for each plane.
 373     planes = p_sys->p_proc_image->i_planes;
 374     p_sys->i_cv_image_size = planes;
 375     for( int i = 0; i < planes; i++ )
 376     {
 377         sz = cvSize(abs(p_sys->p_proc_image->p[i].i_visible_pitch /
 378             p_sys->p_proc_image->p[i].i_pixel_pitch),
 379             abs(p_sys->p_proc_image->p[i].i_visible_lines));
 380
 381         p_sys->p_cv_image[i] = cvCreateImageHeader(sz, IPL_DEPTH_8U,
 382             p_sys->p_proc_image->p[i].i_pixel_pitch);
 383
 384         cvSetData( p_sys->p_cv_image[i],
 385             (char*)(p_sys->p_proc_image->p[i].p_pixels), p_sys->p_proc_image->p[i].i_pitch );
 386     }
 387
 388     //Hack the above opencv image array into a picture_t so that it can be sent to
 389     //another video filter
 390     p_sys->hacked_pic.i_planes = planes;
 391     p_sys->hacked_pic.format.i_chroma = fmt_out.i_chroma;
 392
 393 #ifndef NDEBUG
 394     msg_Dbg( p_filter, "VlcPictureToIplImageRgb() completed" );
 395 #endif
 396 }
 397
 398 /*****************************************************************************
 399  * Filter: displays previously rendered output
 400  *****************************************************************************
 401  * This function send the currently rendered image to the internal opencv
 402  * filter for processing.
 403  *****************************************************************************/
 404 static picture_t* Filter( filter_t* p_filter, picture_t* p_pic )
 405 {
 406     picture_t* p_outpic = filter_NewPicture( p_filter );
 407     if( p_outpic == NULL ) {
 408         msg_Err( p_filter, "couldn't get a p_outpic!" );
 409         picture_Release( p_pic );
 410         return NULL;
 411     }
 412
 413     video_format_t fmt_out;
 414
 415     // Make a copy if we want to show the original input
 416     if (p_filter->p_sys->i_wrapper_output == VINPUT)
 417         picture_Copy( p_outpic, p_pic );
 418
 419     VlcPictureToIplImage( p_filter, p_pic );
 420     // Pass the image (as a pointer to the first IplImage*) to the
 421     // internal OpenCV filter for processing.
 422     p_filter->p_sys->p_opencv->pf_video_filter( p_filter->p_sys->p_opencv, (picture_t*)&(p_filter->p_sys->p_cv_image[0]) );
 423
 424     if(p_filter->p_sys->i_wrapper_output == PROCESSED) {
 425         // Processed video
 426         if( (p_filter->p_sys->p_proc_image) &&
 427             (p_filter->p_sys->p_proc_image->i_planes > 0) &&
 428             (p_filter->p_sys->i_internal_chroma != CINPUT) ) {
 429             //p_filter->p_sys->p_proc_image->format.i_chroma = VLC_CODEC_RGB24;
 430
 431             memset( &fmt_out, 0, sizeof(video_format_t) );
 432             fmt_out = p_pic->format;
 433             //picture_Release( p_outpic );
 434
 435             /*
 436              * We have to copy out the image from image_Convert(), otherwise
 437              * you leak pictures for some reason:
 438              * main video output error: pictures leaked, trying to workaround
 439              */
 440             picture_t* p_outpic_tmp = image_Convert(
 441                         p_filter->p_sys->p_image,
 442                         p_filter->p_sys->p_proc_image,
 443                         &(p_filter->p_sys->p_proc_image->format),
 444                         &fmt_out );
 445
 446             picture_CopyPixels( p_outpic, p_outpic_tmp );
 447             CopyInfoAndRelease( p_outpic, p_outpic_tmp );
 448         } else if( p_filter->p_sys->i_internal_chroma == CINPUT ) {
 449             picture_CopyPixels( p_outpic, p_filter->p_sys->p_proc_image );
 450             picture_CopyProperties( p_outpic, p_filter->p_sys->p_proc_image );
 451         }
 452     }
 453
 454     ReleaseImages( p_filter );
 455     picture_Release( p_pic );
 456
 457 #ifndef NDEBUG
 458     msg_Dbg( p_filter, "Filter() done" );
 459 #endif
 460
 461     if( p_filter->p_sys->i_wrapper_output != NONE ) {
 462         return p_outpic;
 463     } else { // NONE
 464         picture_Release( p_outpic );
 465         return NULL;
 466     }
 467 }
 468