modules/access/avcapture.m

   1 /*****************************************************************************
   2  * avcapture.m: AVFoundation (Mac OS X) based video capture module
   3  *****************************************************************************
   4  * Copyright © 2008-2013 VLC authors and VideoLAN
   5  *
   6  * Authors: Michael Feurstein <michael.feurstein@gmail.com>
   7  *
   8  ****************************************************************************
   9  * This program is free software; you can redistribute it and/or modify it
  10  * under the terms of the GNU Lesser General Public License as published by
  11  * the Free Software Foundation; either version 2.1 of the License, or
  12  * (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  17  * GNU Lesser General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU Lesser General Public License
  20  * along with this program; if not, write to the Free Software Foundation,
  21  * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  22  *****************************************************************************/
  23
  24 /*****************************************************************************
  25  * Preamble
  26  *****************************************************************************/
  27
  28 #define OS_OBJECT_USE_OBJC 0
  29
  30 #ifdef HAVE_CONFIG_H
  31 # include "config.h"
  32 #endif
  33
  34 #include <vlc_common.h>
  35 #include <vlc_plugin.h>
  36 #include <vlc_input.h>
  37 #include <vlc_demux.h>
  38 #include <vlc_interface.h>
  39 #include <vlc_dialog.h>
  40 #include <vlc_access.h>
  41
  42 #import <AVFoundation/AVFoundation.h>
  43 #import <CoreMedia/CoreMedia.h>
  44
  45 /*****************************************************************************
  46 * Local prototypes
  47 *****************************************************************************/
  48 static int Open(vlc_object_t *p_this);
  49 static void Close(vlc_object_t *p_this);
  50 static int Demux(demux_t *p_demux);
  51 static int Control(demux_t *, int, va_list);
  52
  53 /*****************************************************************************
  54 * Module descriptor
  55 *****************************************************************************/
  56 vlc_module_begin ()
  57    set_shortname(N_("AVFoundation Video Capture"))
  58    set_description(N_("AVFoundation video capture module."))
  59    set_category(CAT_INPUT)
  60    set_subcategory(SUBCAT_INPUT_ACCESS)
  61    add_shortcut("avcapture")
  62    set_capability("access_demux", 10)
  63    set_callbacks(Open, Close)
  64 vlc_module_end ()
  65
  66
  67 /*****************************************************************************
  68 * AVFoundation Bridge
  69 *****************************************************************************/
  70 @interface VLCAVDecompressedVideoOutput : AVCaptureVideoDataOutput
  71 {
  72     demux_t             *p_avcapture;
  73
  74     CVImageBufferRef    currentImageBuffer;
  75
  76     mtime_t             currentPts;
  77     mtime_t             previousPts;
  78     size_t              bytesPerRow;
  79
  80     long                timeScale;
  81     BOOL                videoDimensionsReady;
  82 }
  83
  84 @property (readwrite) CMVideoDimensions videoDimensions;
  85
  86 - (id)initWithDemux:(demux_t *)p_demux;
  87 - (int)width;
  88 - (int)height;
  89 - (void)getVideoDimensions:(CMSampleBufferRef)sampleBuffer;
  90 - (mtime_t)currentPts;
  91 - (void)captureOutput:(AVCaptureOutput *)captureOutput didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection;
  92 - (mtime_t)copyCurrentFrameToBuffer:(void *)buffer;
  93 @end
  94
  95 @implementation VLCAVDecompressedVideoOutput : AVCaptureVideoDataOutput
  96
  97 - (id)initWithDemux:(demux_t *)p_demux
  98 {
  99     if (self = [super init])
 100     {
 101         p_avcapture = p_demux;
 102         currentImageBuffer = nil;
 103         currentPts = 0;
 104         previousPts = 0;
 105         bytesPerRow = 0;
 106         timeScale = 0;
 107         videoDimensionsReady = NO;
 108     }
 109     return self;
 110 }
 111
 112 - (void)dealloc
 113 {
 114     @synchronized (self)
 115     {
 116         CVBufferRelease(currentImageBuffer);
 117         currentImageBuffer = nil;
 118         bytesPerRow = 0;
 119         videoDimensionsReady = NO;
 120     }
 121 }
 122
 123 - (long)timeScale
 124 {
 125     return timeScale;
 126 }
 127
 128 - (int)width
 129 {
 130     return self.videoDimensions.width;
 131 }
 132
 133 - (int)height
 134 {
 135     return self.videoDimensions.height;
 136 }
 137
 138 - (size_t)bytesPerRow
 139 {
 140     return bytesPerRow;
 141 }
 142
 143 - (void)getVideoDimensions:(CMSampleBufferRef)sampleBuffer
 144 {
 145     if (!videoDimensionsReady)
 146     {
 147         CMFormatDescriptionRef formatDescription = CMSampleBufferGetFormatDescription(sampleBuffer);
 148         self.videoDimensions = CMVideoFormatDescriptionGetDimensions(formatDescription);
 149         bytesPerRow = CVPixelBufferGetBytesPerRow(CMSampleBufferGetImageBuffer(sampleBuffer));
 150         videoDimensionsReady = YES;
 151         msg_Dbg(p_avcapture, "Dimensionns obtained height:%i width:%i bytesPerRow:%lu", [self height], [self width], bytesPerRow);
 152     }
 153 }
 154
 155 -(mtime_t)currentPts
 156 {
 157     mtime_t pts;
 158
 159     if ( !currentImageBuffer || currentPts == previousPts )
 160         return 0;
 161
 162     @synchronized (self)
 163     {
 164         pts = previousPts = currentPts;
 165     }
 166
 167     return currentPts;
 168 }
 169
 170 - (void)captureOutput:(AVCaptureOutput *)captureOutput
 171 didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
 172        fromConnection:(AVCaptureConnection *)connection
 173 {
 174     @autoreleasepool {
 175         CVImageBufferRef imageBufferToRelease;
 176         CMTime presentationtimestamp = CMSampleBufferGetPresentationTimeStamp(sampleBuffer);
 177         CVImageBufferRef videoFrame = CMSampleBufferGetImageBuffer(sampleBuffer);
 178         CVBufferRetain(videoFrame);
 179         [self getVideoDimensions:sampleBuffer];
 180
 181         @synchronized (self) {
 182             imageBufferToRelease = currentImageBuffer;
 183             currentImageBuffer = videoFrame;
 184             currentPts = (mtime_t)presentationtimestamp.value;
 185             timeScale = (long)presentationtimestamp.timescale;
 186         }
 187
 188         CVBufferRelease(imageBufferToRelease);
 189     }
 190 }
 191
 192 - (mtime_t)copyCurrentFrameToBuffer:(void *)buffer
 193 {
 194     CVImageBufferRef imageBuffer;
 195     mtime_t pts;
 196
 197     void *pixels;
 198
 199     if ( !currentImageBuffer || currentPts == previousPts )
 200         return 0;
 201
 202     @synchronized (self)
 203     {
 204         imageBuffer = CVBufferRetain(currentImageBuffer);
 205         if (imageBuffer)
 206         {
 207             pts = previousPts = currentPts;
 208             CVPixelBufferLockBaseAddress(imageBuffer, 0);
 209             pixels = CVPixelBufferGetBaseAddress(imageBuffer);
 210             if (pixels)
 211             {
 212                 memcpy(buffer, pixels, CVPixelBufferGetHeight(imageBuffer) * CVPixelBufferGetBytesPerRow(imageBuffer));
 213             }
 214             CVPixelBufferUnlockBaseAddress(imageBuffer, 0);
 215         }
 216     }
 217     CVBufferRelease(imageBuffer);
 218
 219     if (pixels)
 220         return currentPts;
 221     else
 222         return 0;
 223 }
 224
 225 @end
 226
 227 /*****************************************************************************
 228 * Struct
 229 *****************************************************************************/
 230
 231 struct demux_sys_t
 232 {
 233     CFTypeRef _Nullable             session;       // AVCaptureSession
 234     CFTypeRef _Nullable             device;        // AVCaptureDevice
 235     CFTypeRef _Nullable             output;        // VLCAVDecompressedVideoOutput
 236     es_out_id_t                     *p_es_video;
 237     es_format_t                     fmt;
 238     int                             height, width;
 239     BOOL                            b_es_setup;
 240 };
 241
 242 /*****************************************************************************
 243 * Open:
 244 *****************************************************************************/
 245 static int Open(vlc_object_t *p_this)
 246 {
 247     demux_t                 *p_demux = (demux_t*)p_this;
 248     demux_sys_t             *p_sys = NULL;
 249
 250     NSString                *avf_currdevice_uid;
 251     NSArray                 *myVideoDevices;
 252     NSError                 *o_returnedError;
 253
 254     AVCaptureDeviceInput    *input = nil;
 255
 256     int                     i, i_width, i_height, deviceCount, ivideo;
 257
 258     char                    *psz_uid = NULL;
 259
 260     /* Only when selected */
 261     if ( *p_demux->psz_access == '\0' )
 262         return VLC_EGENERIC;
 263
 264     @autoreleasepool {
 265         if (p_demux->psz_location && *p_demux->psz_location)
 266             psz_uid = strdup(p_demux->psz_location);
 267
 268         msg_Dbg(p_demux, "avcapture uid = %s", psz_uid);
 269         avf_currdevice_uid = [[NSString alloc] initWithFormat:@"%s", psz_uid];
 270
 271         /* Set up p_demux */
 272         p_demux->pf_demux = Demux;
 273         p_demux->pf_control = Control;
 274         p_demux->info.i_update = 0;
 275         p_demux->info.i_title = 0;
 276         p_demux->info.i_seekpoint = 0;
 277
 278         p_demux->p_sys = p_sys = calloc(1, sizeof(demux_sys_t));
 279         if ( !p_sys )
 280             return VLC_ENOMEM;
 281
 282         myVideoDevices = [[AVCaptureDevice devicesWithMediaType:AVMediaTypeVideo]
 283                            arrayByAddingObjectsFromArray:[AVCaptureDevice devicesWithMediaType:AVMediaTypeMuxed]];
 284         if ( [myVideoDevices count] == 0 )
 285         {
 286             vlc_dialog_display_error(p_demux, _("No video devices found"),
 287                 _("Your Mac does not seem to be equipped with a suitable video input device. "
 288                 "Please check your connectors and drivers."));
 289             msg_Err(p_demux, "Can't find any suitable video device");
 290             goto error;
 291         }
 292
 293         deviceCount = [myVideoDevices count];
 294         for ( ivideo = 0; ivideo < deviceCount; ivideo++ )
 295         {
 296             AVCaptureDevice *avf_device;
 297             avf_device = [myVideoDevices objectAtIndex:ivideo];
 298             msg_Dbg(p_demux, "avcapture %i/%i %s %s", ivideo, deviceCount, [[avf_device modelID] UTF8String], [[avf_device uniqueID] UTF8String]);
 299             if ([[[avf_device uniqueID]stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceCharacterSet]] isEqualToString:avf_currdevice_uid]) {
 300                 break;
 301             }
 302         }
 303
 304         if ( ivideo < [myVideoDevices count] )
 305         {
 306             p_sys->device = CFBridgingRetain([myVideoDevices objectAtIndex:ivideo]);
 307         }
 308         else
 309         {
 310             msg_Dbg(p_demux, "Cannot find designated device as %s, falling back to default.", [avf_currdevice_uid UTF8String]);
 311             p_sys->device = CFBridgingRetain([AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeVideo]);
 312         }
 313         if ( !p_sys->device )
 314         {
 315             vlc_dialog_display_error(p_demux, _("No video devices found"),
 316                 _("Your Mac does not seem to be equipped with a suitable input device. "
 317                 "Please check your connectors and drivers."));
 318             msg_Err(p_demux, "Can't find any suitable video device");
 319             goto error;
 320         }
 321
 322         if ( [(__bridge AVCaptureDevice *)p_sys->device isInUseByAnotherApplication] == YES )
 323         {
 324             msg_Err(p_demux, "default capture device is exclusively in use by another application");
 325             goto error;
 326         }
 327
 328         input = [AVCaptureDeviceInput deviceInputWithDevice:(__bridge AVCaptureDevice *)p_sys->device error:&o_returnedError];
 329
 330         if ( !input )
 331         {
 332             msg_Err(p_demux, "can't create a valid capture input facility (%ld)", [o_returnedError code]);
 333             goto error;
 334         }
 335
 336         int chroma = VLC_CODEC_RGB32;
 337
 338         memset(&p_sys->fmt, 0, sizeof(es_format_t));
 339         es_format_Init(&p_sys->fmt, VIDEO_ES, chroma);
 340
 341         p_sys->session = CFBridgingRetain([[AVCaptureSession alloc] init]);
 342         [(__bridge AVCaptureSession *)p_sys->session addInput:input];
 343
 344         p_sys->output = CFBridgingRetain([[VLCAVDecompressedVideoOutput alloc] initWithDemux:p_demux]);
 345         [(__bridge AVCaptureSession *)p_sys->session addOutput:(__bridge VLCAVDecompressedVideoOutput *)p_sys->output];
 346
 347         dispatch_queue_t queue = dispatch_queue_create("avCaptureQueue", NULL);
 348         [(__bridge VLCAVDecompressedVideoOutput *)p_sys->output setSampleBufferDelegate:(__bridge id)p_sys->output queue:queue];
 349         dispatch_release(queue);
 350
 351         [(__bridge VLCAVDecompressedVideoOutput *)p_sys->output setVideoSettings:[NSDictionary dictionaryWithObject:[NSNumber numberWithInt:kCVPixelFormatType_32BGRA] forKey:(id)kCVPixelBufferPixelFormatTypeKey]];
 352         [(__bridge AVCaptureSession *)p_sys->session startRunning];
 353
 354         input = nil;
 355
 356         msg_Dbg(p_demux, "AVCapture: Video device ready!");
 357
 358         return VLC_SUCCESS;
 359     error:
 360         msg_Err(p_demux, "Error");
 361         input = nil;
 362
 363         free(p_sys);
 364
 365         return VLC_EGENERIC;
 366     }
 367 }
 368
 369 /*****************************************************************************
 370 * Close:
 371 *****************************************************************************/
 372 static void Close(vlc_object_t *p_this)
 373 {
 374     demux_t             *p_demux = (demux_t*)p_this;
 375     demux_sys_t         *p_sys = p_demux->p_sys;
 376
 377     @autoreleasepool {
 378         msg_Dbg(p_demux,"Close AVCapture");
 379
 380         // Perform this on main thread, as the framework itself will sometimes try to synchronously
 381         // work on main thread. And this will create a dead lock.
 382         [(__bridge AVCaptureSession *)p_sys->session performSelectorOnMainThread:@selector(stopRunning) withObject:nil waitUntilDone:NO];
 383         CFBridgingRelease(p_sys->output);
 384         CFBridgingRelease(p_sys->session);
 385
 386         free(p_sys);
 387     }
 388 }
 389
 390 /*****************************************************************************
 391 * Demux:
 392 *****************************************************************************/
 393 static int Demux(demux_t *p_demux)
 394 {
 395     demux_sys_t *p_sys = p_demux->p_sys;
 396     block_t     *p_block;
 397
 398     @autoreleasepool {
 399         @synchronized ( p_sys->output )
 400         {
 401             p_block = block_Alloc([(__bridge VLCAVDecompressedVideoOutput *)p_sys->output width] * [(__bridge VLCAVDecompressedVideoOutput *)p_sys->output bytesPerRow]);
 402
 403             if ( !p_block )
 404             {
 405                 msg_Err(p_demux, "cannot get block");
 406                 return 0;
 407             }
 408
 409             p_block->i_pts = [(__bridge VLCAVDecompressedVideoOutput *)p_sys->output copyCurrentFrameToBuffer: p_block->p_buffer];
 410
 411             if ( !p_block->i_pts )
 412             {
 413                 /* Nothing to display yet, just forget */
 414                 block_Release(p_block);
 415                 msleep(10000);
 416                 return 1;
 417             }
 418             else if ( !p_sys->b_es_setup )
 419             {
 420                 p_sys->fmt.video.i_frame_rate_base = [(__bridge VLCAVDecompressedVideoOutput *)p_sys->output timeScale];
 421                 msg_Dbg(p_demux, "using frame rate base: %i", p_sys->fmt.video.i_frame_rate_base);
 422                 p_sys->width = p_sys->fmt.video.i_width = [(__bridge VLCAVDecompressedVideoOutput *)p_sys->output width];
 423                 p_sys->height = p_sys->fmt.video.i_height = [(__bridge VLCAVDecompressedVideoOutput *)p_sys->output height];
 424                 p_sys->p_es_video = es_out_Add(p_demux->out, &p_sys->fmt);
 425                 msg_Dbg(p_demux, "added new video es %4.4s %dx%d", (char*)&p_sys->fmt.i_codec, p_sys->width, p_sys->height);
 426                 p_sys->b_es_setup = YES;
 427             }
 428         }
 429
 430         es_out_Control(p_demux->out, ES_OUT_SET_PCR, p_block->i_pts);
 431         es_out_Send(p_demux->out, p_sys->p_es_video, p_block);
 432
 433     }
 434     return 1;
 435 }
 436
 437 /*****************************************************************************
 438 * Control:
 439 *****************************************************************************/
 440 static int Control(demux_t *p_demux, int i_query, va_list args)
 441 {
 442     bool        *pb;
 443     int64_t     *pi64;
 444
 445     switch( i_query )
 446     {
 447         /* Special for access_demux */
 448         case DEMUX_CAN_PAUSE:
 449         case DEMUX_CAN_SEEK:
 450         case DEMUX_SET_PAUSE_STATE:
 451         case DEMUX_CAN_CONTROL_PACE:
 452            pb = va_arg(args, bool *);
 453            *pb = false;
 454            return VLC_SUCCESS;
 455
 456         case DEMUX_GET_PTS_DELAY:
 457            pi64 = va_arg(args, int64_t *);
 458            *pi64 = INT64_C(1000) * var_InheritInteger(p_demux, "live-caching");
 459            return VLC_SUCCESS;
 460
 461         case DEMUX_GET_TIME:
 462             pi64 = va_arg(args, int64_t *);
 463             *pi64 = mdate();
 464             return VLC_SUCCESS;
 465
 466         default:
 467            return VLC_EGENERIC;
 468     }
 469     return VLC_EGENERIC;
 470 }