modules/access/avaudiocapture.m

   1 /*****************************************************************************
   2  * avaudiocapture.m: AVFoundation based audio capture module
   3  *****************************************************************************
   4  * Copyright © 2018 VLC authors and VideoLAN
   5  *
   6  * Authors: Pierre d'Herbemont <pdherbemont@videolan.org>
   7  *          Gustaf Neumann <neumann@wu.ac.at>
   8  *          Michael S. Feurstein <michael.feurstein@wu.ac.at>
   9  *          David Fuhrmann <dfuhrmann at videolan dot org>
  10  *
  11  ****************************************************************************
  12  * This program is free software; you can redistribute it and/or modify it
  13  * under the terms of the GNU Lesser General Public License as published by
  14  * the Free Software Foundation; either version 2.1 of the License, or
  15  * (at your option) any later version.
  16  *
  17  * This program is distributed in the hope that it will be useful,
  18  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  19  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  20  * GNU Lesser General Public License for more details.
  21  *
  22  * You should have received a copy of the GNU Lesser General Public License
  23  * along with this program; if not, write to the Free Software Foundation,
  24  * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  25  *****************************************************************************/
  26
  27 /*****************************************************************************
  28  * Preamble
  29  *****************************************************************************/
  30
  31 #ifdef HAVE_CONFIG_H
  32 # include "config.h"
  33 #endif
  34
  35 #include <assert.h>
  36
  37 #include <vlc_common.h>
  38 #include <vlc_plugin.h>
  39 #include <vlc_input.h>
  40 #include <vlc_demux.h>
  41 #include <vlc_dialog.h>
  42
  43 #import <AvailabilityMacros.h>
  44 #import <AVFoundation/AVFoundation.h>
  45 #import <CoreMedia/CoreMedia.h>
  46
  47
  48 #ifndef MAC_OS_X_VERSION_10_14
  49 @interface AVCaptureDevice (AVCaptureDeviceAuthorizationSince10_14)
  50
  51 + (void)requestAccessForMediaType:(AVMediaType)mediaType completionHandler:(void (^)(BOOL granted))handler API_AVAILABLE(macos(10.14), ios(7.0));
  52
  53 @end
  54 #endif
  55
  56 /*****************************************************************************
  57  * Struct
  58  *****************************************************************************/
  59
  60 typedef struct demux_sys_t
  61 {
  62     CFTypeRef _Nullable             session;       // AVCaptureSession
  63     es_out_id_t                     *p_es_audio;
  64
  65 } demux_sys_t;
  66
  67
  68 /*****************************************************************************
  69 * AVFoundation Bridge
  70 *****************************************************************************/
  71 @interface VLCAVDecompressedAudioOutput : AVCaptureAudioDataOutput<AVCaptureAudioDataOutputSampleBufferDelegate>
  72 {
  73     demux_t *p_avcapture;
  74     date_t date;
  75 }
  76
  77 @end
  78
  79 @implementation VLCAVDecompressedAudioOutput : AVCaptureAudioDataOutput
  80
  81 - (id)initWithDemux:(demux_t *)p_demux
  82 {
  83     if (self = [super init])
  84     {
  85         p_avcapture = p_demux;
  86
  87         date_Init(&date, 44100, 1);
  88         date_Set(&date, VLC_TICK_0);
  89     }
  90     return self;
  91 }
  92
  93 - (void)captureOutput:(AVCaptureOutput *)captureOutput
  94 didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer
  95        fromConnection:(AVCaptureConnection *)connection
  96 {
  97     @autoreleasepool {
  98         CMItemCount numSamplesInBuffer = CMSampleBufferGetNumSamples(sampleBuffer);
  99
 100         size_t neededBufferListSize = 0;
 101         // first get needed size for buffer
 102         OSStatus retValue = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(sampleBuffer, &neededBufferListSize, nil, 0, nil, nil, kCMSampleBufferFlag_AudioBufferList_Assure16ByteAlignment, nil);
 103
 104         if (retValue != noErr) {
 105             msg_Err(p_avcapture, "Error getting sample list buffer size: %d", retValue);
 106             return;
 107         }
 108
 109         CMBlockBufferRef blockBuffer;
 110         AudioBufferList *audioBufferList = calloc(1, neededBufferListSize);
 111         retValue = CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(sampleBuffer, nil, audioBufferList, neededBufferListSize, nil, nil, kCMSampleBufferFlag_AudioBufferList_Assure16ByteAlignment, &blockBuffer);
 112
 113         if (retValue != noErr) {
 114             msg_Err(p_avcapture, "Cannot get samples from buffer: %d", retValue);
 115             return;
 116         }
 117
 118         if (audioBufferList->mNumberBuffers != 1) {
 119             msg_Warn(p_avcapture, "This module expects 1 buffer only, got %d", audioBufferList->mNumberBuffers);
 120             return;
 121         }
 122
 123         int64_t totalDataSize = audioBufferList->mBuffers[0].mDataByteSize;
 124         block_t *outBlock = block_Alloc(totalDataSize);
 125         if (!outBlock)
 126             return;
 127
 128         date_Increment(&date, (uint32_t)numSamplesInBuffer);
 129
 130         memcpy(outBlock->p_buffer, audioBufferList->mBuffers[0].mData, totalDataSize);
 131         outBlock->i_pts = date_Get(&date);
 132         outBlock->i_nb_samples = (unsigned)numSamplesInBuffer;
 133
 134         CFRelease(blockBuffer);
 135
 136         demux_sys_t *p_sys = p_avcapture->p_sys;
 137         es_out_SetPCR(p_avcapture->out, outBlock->i_pts);
 138         es_out_Send(p_avcapture->out, p_sys->p_es_audio, outBlock);
 139     }
 140 }
 141
 142 @end
 143
 144
 145 /*****************************************************************************
 146  * Control:
 147  *****************************************************************************/
 148 static int Control(demux_t *p_demux, int i_query, va_list args)
 149 {
 150     bool *pb;
 151
 152     switch(i_query) {
 153             /* Special for access_demux */
 154         case DEMUX_CAN_PAUSE:
 155         case DEMUX_CAN_SEEK:
 156         case DEMUX_SET_PAUSE_STATE:
 157         case DEMUX_CAN_CONTROL_PACE:
 158             pb = (bool*)va_arg(args, bool *);
 159             *pb = false;
 160             return VLC_SUCCESS;
 161
 162         case DEMUX_GET_PTS_DELAY:
 163             *va_arg(args, vlc_tick_t *) =
 164             VLC_TICK_FROM_MS(var_InheritInteger(p_demux, "live-caching"));
 165             return VLC_SUCCESS;
 166
 167         default:
 168             return VLC_EGENERIC;
 169     }
 170     return VLC_EGENERIC;
 171 }
 172
 173 /*****************************************************************************
 174 * Open:
 175 *****************************************************************************/
 176 static int Open(vlc_object_t *p_this)
 177 {
 178     demux_t *p_demux = (demux_t*)p_this;
 179
 180     if (p_demux->out == NULL)
 181         return VLC_EGENERIC;
 182
 183     @autoreleasepool {
 184         NSString *currentDeviceId = @"";
 185         if (p_demux->psz_location && *p_demux->psz_location)
 186             currentDeviceId = [NSString stringWithUTF8String:p_demux->psz_location];
 187
 188         msg_Dbg(p_demux, "avcapture uid = %s", currentDeviceId.UTF8String);
 189
 190         NSArray *knownAudioDevices = [AVCaptureDevice devicesWithMediaType:AVMediaTypeAudio];
 191         NSInteger numberOfKnownAudioDevices = [knownAudioDevices count];
 192
 193         int selectedDevice = 0;
 194         for (;selectedDevice < numberOfKnownAudioDevices; selectedDevice++ )
 195         {
 196             AVCaptureDevice *avf_device = [knownAudioDevices objectAtIndex:selectedDevice];
 197             msg_Dbg(p_demux, "avcapture %i/%ld %s %s", selectedDevice, (long)numberOfKnownAudioDevices, [[avf_device modelID] UTF8String], [[avf_device uniqueID] UTF8String]);
 198             if ([[[avf_device uniqueID] stringByTrimmingCharactersInSet:[NSCharacterSet whitespaceCharacterSet]] isEqualToString:currentDeviceId]) {
 199                 break;
 200             }
 201         }
 202
 203         AVCaptureDevice *device = nil;
 204         if (selectedDevice < numberOfKnownAudioDevices) {
 205             device = [knownAudioDevices objectAtIndex:selectedDevice];
 206         } else {
 207             msg_Dbg(p_demux, "Cannot find designated device as %s, falling back to default.", currentDeviceId.UTF8String);
 208             device = [AVCaptureDevice defaultDeviceWithMediaType:AVMediaTypeAudio];
 209         }
 210
 211         if (!device) {
 212             vlc_dialog_display_error(p_demux, _("No Audio Input device found"),
 213                                      _("Your Mac does not seem to be equipped with a suitable audio input device."
 214                                        "Please check your connectors and drivers."));
 215             msg_Err(p_demux, "Can't find any Audio device");
 216             return VLC_EGENERIC;
 217         }
 218
 219         if ([device isInUseByAnotherApplication]) {
 220             msg_Err(p_demux, "Capture device is exclusively in use by another application");
 221             return VLC_EGENERIC;
 222         }
 223
 224         if (@available(macOS 10.14, *)) {
 225             msg_Dbg(p_demux, "Check user consent for access to the audio device");
 226
 227             dispatch_semaphore_t sema = dispatch_semaphore_create(0);
 228             __block bool accessGranted = NO;
 229             [AVCaptureDevice requestAccessForMediaType: AVMediaTypeAudio completionHandler:^(BOOL granted) {
 230                 accessGranted = granted;
 231                 dispatch_semaphore_signal(sema);
 232             } ];
 233             dispatch_semaphore_wait(sema, DISPATCH_TIME_FOREVER);
 234             if (!accessGranted) {
 235                 msg_Err(p_demux, "Can't use the audio device as access has not been granted by the user");
 236                 vlc_dialog_display_error(p_demux, _("Problem accessing a system resource"),
 237                     _("Please open \"System Preferences\" -> \"Security & Privacy\" "
 238                       "and allow VLC to access your microphone."));
 239
 240                 return VLC_EGENERIC;
 241             }
 242         }
 243
 244         NSError *error = nil;
 245         AVCaptureDeviceInput *input = [AVCaptureDeviceInput deviceInputWithDevice:device error:&error];
 246         if (!input) {
 247             msg_Err(p_demux, "can't create a valid capture input facility (%ld)", [error code]);
 248             return VLC_EGENERIC;
 249         }
 250
 251         /* Now we can init */
 252         int audiocodec = VLC_CODEC_FL32;
 253         es_format_t audiofmt;
 254         es_format_Init(&audiofmt, AUDIO_ES, audiocodec);
 255
 256         audiofmt.audio.i_format = audiocodec;
 257         audiofmt.audio.i_rate = 44100;
 258         /*
 259          * i_physical_channels Describes the channels configuration of the
 260          * samples (ie. number of channels which are available in the
 261          * buffer, and positions).
 262          */
 263         audiofmt.audio.i_physical_channels = AOUT_CHAN_RIGHT | AOUT_CHAN_LEFT;
 264         /*
 265          * Please note that it may be completely arbitrary - buffers are not
 266          * obliged to contain a integral number of so-called "frames". It's
 267          * just here for the division:
 268          * buffer_size = i_nb_samples * i_bytes_per_frame / i_frame_length
 269          */
 270         audiofmt.audio.i_bitspersample = 32;
 271         audiofmt.audio.i_channels = 2;
 272         audiofmt.audio.i_blockalign = audiofmt.audio.i_channels * (audiofmt.audio.i_bitspersample / 8);
 273         audiofmt.i_bitrate = audiofmt.audio.i_channels * audiofmt.audio.i_rate * audiofmt.audio.i_bitspersample;
 274
 275
 276         AVCaptureSession *session = [[AVCaptureSession alloc] init];
 277         [session addInput:input];
 278
 279         VLCAVDecompressedAudioOutput *output = [[VLCAVDecompressedAudioOutput alloc] initWithDemux:p_demux];
 280         [session addOutput:output];
 281
 282         dispatch_queue_t queue = dispatch_queue_create("avCaptureQueue", NULL);
 283         [output setSampleBufferDelegate:output queue:queue];
 284
 285         [output setAudioSettings:
 286             @{
 287               AVFormatIDKey : @(kAudioFormatLinearPCM),
 288               AVLinearPCMBitDepthKey : @(32),
 289               AVLinearPCMIsFloatKey : @YES,
 290               AVLinearPCMIsBigEndianKey : @NO,
 291               AVNumberOfChannelsKey : @(2),
 292               AVLinearPCMIsNonInterleaved : @NO,
 293               AVSampleRateKey : @(44100.0)
 294             }];
 295
 296         /* Set up p_demux */
 297         p_demux->pf_demux = NULL;
 298         p_demux->pf_control = Control;
 299
 300         demux_sys_t *p_sys = NULL;
 301         p_demux->p_sys = p_sys = calloc(1, sizeof(demux_sys_t));
 302         if (!p_sys)
 303             return VLC_ENOMEM;
 304
 305         p_sys->session = CFBridgingRetain(session);
 306         p_sys->p_es_audio = es_out_Add(p_demux->out, &audiofmt);
 307
 308         [session startRunning];
 309         msg_Dbg(p_demux, "AVCapture: Audio device ready!");
 310         return VLC_SUCCESS;
 311     }
 312 }
 313
 314 /*****************************************************************************
 315 * Close:
 316 *****************************************************************************/
 317 static void Close(vlc_object_t *p_this)
 318 {
 319     demux_t             *p_demux = (demux_t*)p_this;
 320     demux_sys_t         *p_sys = p_demux->p_sys;
 321
 322     @autoreleasepool {
 323         msg_Dbg(p_demux,"Close AVCapture");
 324
 325         ///@todo Investigate why this should be needed
 326         // Perform this on main thread, as the framework itself will sometimes try to synchronously
 327         // work on main thread. And this will create a dead lock.
 328 //        [(__bridge AVCaptureSession *)p_sys->session performSelectorOnMainThread:@selector(stopRunning) withObject:nil waitUntilDone:YES];
 329
 330         [(__bridge AVCaptureSession *)p_sys->session stopRunning];
 331         CFBridgingRelease(p_sys->session);
 332
 333         free(p_sys);
 334     }
 335 }
 336
 337 /*****************************************************************************
 338  * Module descriptor
 339  *****************************************************************************/
 340 vlc_module_begin ()
 341 set_shortname(N_("AVFoundation Audio Capture"))
 342 set_description(N_("AVFoundation audio capture module."))
 343 set_category(CAT_INPUT)
 344 set_subcategory(SUBCAT_INPUT_ACCESS)
 345 add_shortcut("qtsound")
 346 set_capability("access", 0)
 347 set_callbacks(Open, Close)
 348 vlc_module_end ()