modules/codec/audiotoolbox_midi.c

   1 /*****************************************************************************
   2  * audiotoolbox_midi.c: Software MIDI synthesizer using AudioToolbox
   3  *****************************************************************************
   4  * Copyright (C) 2017 VLC authors and VideoLAN
   5  *
   6  * Authors: Marvin Scholz <epirat07 at gmail dot com>
   7  *
   8  * Based on the fluidsynth module by Rémi Denis-Courmont
   9  *
  10  * This program is free software; you can redistribute it and/or modify it
  11  * under the terms of the GNU Lesser General Public License as published by
  12  * the Free Software Foundation; either version 2.1 of the License, or
  13  * (at your option) any later version.
  14  *
  15  * This program is distributed in the hope that it will be useful,
  16  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  17  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  18  * GNU Lesser General Public License for more details.
  19  *
  20  * You should have received a copy of the GNU Lesser General Public License
  21  * along with this program; if not, write to the Free Software Foundation,
  22  * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  23  *****************************************************************************/
  24
  25 #ifdef HAVE_CONFIG_H
  26 # include "config.h"
  27 #endif
  28
  29 #include <vlc_common.h>
  30 #include <vlc_plugin.h>
  31 #include <vlc_codec.h>
  32 #include <vlc_dialog.h>
  33
  34 #include <CoreFoundation/CoreFoundation.h>
  35 #include <AudioUnit/AudioUnit.h>
  36 #include <AudioToolbox/AudioToolbox.h>
  37
  38 #include <TargetConditionals.h>
  39
  40 #ifndef on_err_goto
  41 #define on_err_goto(errorCode, exceptionLabel)  \
  42 do { if ((errorCode) != noErr) goto exceptionLabel; \
  43 } while ( 0 )
  44 #endif
  45
  46 #define SOUNDFONT_TEXT N_("SoundFont file")
  47 #define SOUNDFONT_LONGTEXT N_( \
  48     "SoundFont file to use for software synthesis." )
  49
  50 static int  Open  (vlc_object_t *);
  51 static void Close (vlc_object_t *);
  52
  53 #define CFG_PREFIX "aumidi-"
  54
  55 vlc_module_begin()
  56     set_description(N_("AudioToolbox MIDI synthesizer"))
  57     set_capability("audio decoder", 100)
  58     set_shortname(N_("AUMIDI"))
  59     set_category(CAT_INPUT)
  60     set_subcategory(SUBCAT_INPUT_ACODEC)
  61     set_callbacks(Open, Close)
  62     add_loadfile(CFG_PREFIX "soundfont", "",
  63                  SOUNDFONT_TEXT, SOUNDFONT_LONGTEXT)
  64 vlc_module_end()
  65
  66
  67 typedef struct
  68 {
  69     AUGraph     graph;
  70     AudioUnit   synthUnit;
  71     AudioUnit   outputUnit;
  72     date_t       end_date;
  73 } decoder_sys_t;
  74
  75 static int  DecodeBlock (decoder_t *p_dec, block_t *p_block);
  76 static void Flush (decoder_t *);
  77
  78 /* MIDI constants */
  79 enum
  80 {
  81     kMidiMessage_NoteOff            = 0x80,
  82     kMidiMessage_NoteOn             = 0x90,
  83     kMidiMessage_PolyPressure       = 0xA0,
  84     kMidiMessage_ControlChange      = 0xB0,
  85     kMidiMessage_ProgramChange      = 0xC0,
  86     kMidiMessage_ChannelPressure    = 0xD0,
  87     kMidiMessage_PitchWheel         = 0xE0,
  88     kMidiMessage_SysEx              = 0xF0,
  89
  90     kMidiMessage_BankMSBControl     = 0,
  91     kMidiMessage_BankLSBControl     = 32,
  92
  93     /* Values for kMidiMessage_ControlChange */
  94     kMidiController_AllSoundOff         = 0x78,
  95     kMidiController_ResetAllControllers = 0x79,
  96     kMidiController_AllNotesOff         = 0x7B
  97 };
  98
  99 /* Helper functions */
 100 static OSStatus AddAppleAUNode(AUGraph graph, OSType type, OSType subtype, AUNode *node)
 101 {
 102     AudioComponentDescription cDesc = {};
 103     cDesc.componentType = type;
 104     cDesc.componentSubType = subtype;
 105     cDesc.componentManufacturer = kAudioUnitManufacturer_Apple;
 106
 107     return AUGraphAddNode(graph, &cDesc, node);
 108 }
 109
 110 static OSStatus CreateAUGraph(AUGraph *outGraph, AudioUnit *outSynth, AudioUnit *outOut)
 111 {
 112     OSStatus res;
 113
 114     // AudioUnit nodes
 115     AUNode synthNode, limiterNode, outNode;
 116
 117     // Create the Graph to which we will add our nodes
 118     on_err_goto(res = NewAUGraph(outGraph), bailout);
 119
 120     // Create/add the MIDI synthesizer node (DLS Synth)
 121 #if TARGET_OS_IPHONE
 122     // On iOS/tvOS use MIDISynth, DLSSynth does not exist there
 123     on_err_goto(res = AddAppleAUNode(*outGraph,
 124                                      kAudioUnitType_MusicDevice,
 125                                      kAudioUnitSubType_MIDISynth,
 126                                      &synthNode), bailout);
 127 #else
 128     // Prefer DLSSynth on macOS, as it has a better default behavior
 129     on_err_goto(res = AddAppleAUNode(*outGraph,
 130                                      kAudioUnitType_MusicDevice,
 131                                      kAudioUnitSubType_DLSSynth,
 132                                      &synthNode), bailout);
 133 #endif
 134
 135     // Create/add the peak limiter node
 136     on_err_goto(res = AddAppleAUNode(*outGraph,
 137                                      kAudioUnitType_Effect,
 138                                      kAudioUnitSubType_PeakLimiter,
 139                                      &limiterNode), bailout);
 140
 141     // Create/add the output node (GenericOutput)
 142     on_err_goto(res = AddAppleAUNode(*outGraph,
 143                                      kAudioUnitType_Output,
 144                                      kAudioUnitSubType_GenericOutput,
 145                                      &outNode), bailout);
 146
 147     // Open the Graph, this opens the units that belong to the graph
 148     // so that we can connect them
 149     on_err_goto(res = AUGraphOpen(*outGraph), bailout);
 150
 151     // Connect the synthesizer node to the limiter
 152     on_err_goto(res = AUGraphConnectNodeInput(*outGraph, synthNode, 0, limiterNode, 0), bailout);
 153     // Connect the limiter node to the output
 154     on_err_goto(res = AUGraphConnectNodeInput(*outGraph, limiterNode, 0, outNode, 0), bailout);
 155
 156     // Get reference to the synthesizer node
 157     on_err_goto(res = AUGraphNodeInfo(*outGraph, synthNode, 0, outSynth), bailout);
 158     // Get reference to the output node
 159     on_err_goto(res = AUGraphNodeInfo(*outGraph, outNode, 0, outOut), bailout);
 160
 161 bailout:
 162     return res;
 163 }
 164
 165 static int SetSoundfont(decoder_t *p_dec, AudioUnit synthUnit, const char *sfPath) {
 166     if (!sfPath) {
 167         msg_Dbg(p_dec, "using default soundfont");
 168         return VLC_SUCCESS;
 169     }
 170
 171     msg_Dbg(p_dec, "using custom soundfont: '%s'", sfPath);
 172     CFURLRef url = CFURLCreateFromFileSystemRepresentation(kCFAllocatorDefault,
 173                                                            (const UInt8 *)sfPath,
 174                                                            strlen(sfPath), false);
 175     if (unlikely(url == NULL))
 176         return VLC_ENOMEM;
 177
 178     OSStatus status = AudioUnitSetProperty(synthUnit,
 179                                            kMusicDeviceProperty_SoundBankURL,
 180                                            kAudioUnitScope_Global, 0,
 181                                            &url, sizeof(url));
 182     CFRelease(url);
 183
 184     if (status != noErr) {
 185         msg_Err(p_dec, "failed setting custom SoundFont for MIDI synthesis (%i)", status);
 186         return VLC_EGENERIC;
 187     }
 188     return VLC_SUCCESS;
 189 }
 190
 191 static int Open(vlc_object_t *p_this)
 192 {
 193     decoder_t *p_dec = (decoder_t *)p_this;
 194     OSStatus status = noErr;
 195     int ret = VLC_SUCCESS;
 196
 197     if (p_dec->fmt_in.i_codec != VLC_CODEC_MIDI)
 198         return VLC_EGENERIC;
 199
 200     decoder_sys_t *p_sys = malloc(sizeof (*p_sys));
 201     if (unlikely(p_sys == NULL))
 202         return VLC_ENOMEM;
 203
 204     p_sys->graph = NULL;
 205     status = CreateAUGraph(&p_sys->graph, &p_sys->synthUnit, &p_sys->outputUnit);
 206     if (unlikely(status != noErr)) {
 207         msg_Err(p_dec, "failed to create audiograph (%i)", status);
 208         ret = VLC_EGENERIC;
 209         goto bailout;
 210     }
 211
 212     // Set custom soundfont
 213     char *sfPath = var_InheritString(p_dec, CFG_PREFIX "soundfont");
 214     ret = SetSoundfont(p_dec, p_sys->synthUnit, sfPath);
 215     free(sfPath);
 216     if (unlikely(ret != VLC_SUCCESS))
 217         goto bailout;
 218
 219     // Set VLC output audio format info
 220     p_dec->fmt_out.i_codec = VLC_CODEC_FL32;
 221     p_dec->fmt_out.audio.i_bitspersample = 32;
 222     p_dec->fmt_out.audio.i_rate = 44100;
 223     p_dec->fmt_out.audio.i_channels = 2;
 224     p_dec->fmt_out.audio.i_physical_channels = AOUT_CHAN_LEFT | AOUT_CHAN_RIGHT;
 225
 226     if (decoder_UpdateAudioFormat(p_dec) < 0) {
 227         ret = VLC_EGENERIC;
 228         goto bailout;
 229     }
 230
 231     // Prepare AudioUnit output audio format info
 232     AudioStreamBasicDescription ASBD = {};
 233     unsigned bytesPerSample = sizeof(Float32);
 234     ASBD.mFormatID = kAudioFormatLinearPCM;
 235     ASBD.mFormatFlags = kAudioFormatFlagIsFloat | kAudioFormatFlagIsPacked;
 236     ASBD.mSampleRate = 44100;
 237     ASBD.mFramesPerPacket = 1;
 238     ASBD.mChannelsPerFrame = 2;
 239     ASBD.mBytesPerFrame = bytesPerSample * ASBD.mChannelsPerFrame;
 240     ASBD.mBytesPerPacket = ASBD.mBytesPerFrame * ASBD.mFramesPerPacket;
 241     ASBD.mBitsPerChannel = 8 * bytesPerSample;
 242
 243     // Set AudioUnit format
 244     status = AudioUnitSetProperty(p_sys->outputUnit,
 245                                   kAudioUnitProperty_StreamFormat,
 246                                   kAudioUnitScope_Output, 0, &ASBD,
 247                                   sizeof(AudioStreamBasicDescription));
 248     if (unlikely(status != noErr)) {
 249         msg_Err(p_dec, "failed setting output format for output unit (%i)", status);
 250         ret = VLC_EGENERIC;
 251         goto bailout;
 252     }
 253
 254     // Prepare the AU
 255     status = AUGraphInitialize (p_sys->graph);
 256     if (unlikely(status != noErr)) {
 257         if (status == kAudioUnitErr_InvalidFile)
 258             msg_Err(p_dec, "failed initializing audiograph: invalid soundfont file");
 259         else
 260             msg_Err(p_dec, "failed initializing audiograph (%i)", status);
 261         ret = VLC_EGENERIC;
 262         goto bailout;
 263     }
 264
 265     // Prepare MIDI soundbank
 266     MusicDeviceMIDIEvent(p_sys->synthUnit,
 267                          kMidiMessage_ControlChange,
 268                          kMidiMessage_BankMSBControl, 0, 0);
 269
 270     // Start the AU
 271     status = AUGraphStart(p_sys->graph);
 272     if (unlikely(status != noErr)) {
 273         msg_Err(p_dec, "failed starting audiograph (%i)", status);
 274         ret = VLC_EGENERIC;
 275         goto bailout;
 276     }
 277
 278     // Initialize date (for PTS)
 279     date_Init(&p_sys->end_date, p_dec->fmt_out.audio.i_rate, 1);
 280
 281     p_dec->p_sys = p_sys;
 282     p_dec->pf_decode = DecodeBlock;
 283     p_dec->pf_flush  = Flush;
 284
 285 bailout:
 286     // Cleanup if error occured
 287     if (ret != VLC_SUCCESS) {
 288         if (p_sys->graph)
 289             DisposeAUGraph(p_sys->graph);
 290         free(p_sys);
 291     }
 292     return ret;
 293 }
 294
 295
 296 static void Close (vlc_object_t *p_this)
 297 {
 298     decoder_sys_t *p_sys = ((decoder_t *)p_this)->p_sys;
 299
 300     if (p_sys->graph) {
 301         AUGraphStop(p_sys->graph);
 302         DisposeAUGraph(p_sys->graph);
 303     }
 304     free(p_sys);
 305 }
 306
 307 static void Flush (decoder_t *p_dec)
 308 {
 309     decoder_sys_t *p_sys = p_dec->p_sys;
 310
 311     date_Set(&p_sys->end_date, VLC_TICK_INVALID);
 312
 313     // Turn all sound on all channels off
 314     // else 'old' notes could still be playing
 315     for (unsigned channel = 0; channel < 16; channel++) {
 316         MusicDeviceMIDIEvent(p_sys->synthUnit, kMidiMessage_ControlChange | channel, kMidiController_AllSoundOff, 0, 0);
 317     }
 318 }
 319
 320 static int DecodeBlock (decoder_t *p_dec, block_t *p_block)
 321 {
 322     decoder_sys_t *p_sys = p_dec->p_sys;
 323     block_t *p_out = NULL;
 324     OSStatus status = noErr;
 325
 326     if (p_block == NULL) /* No Drain */
 327         return VLCDEC_SUCCESS;
 328
 329     if (p_block->i_flags & (BLOCK_FLAG_DISCONTINUITY|BLOCK_FLAG_CORRUPTED)) {
 330         Flush(p_dec);
 331         if (p_block->i_flags & BLOCK_FLAG_CORRUPTED) {
 332             block_Release(p_block);
 333             return VLCDEC_SUCCESS;
 334         }
 335     }
 336
 337     if ( p_block->i_pts != VLC_TICK_INVALID &&
 338          date_Get(&p_sys->end_date) == VLC_TICK_INVALID ) {
 339         date_Set(&p_sys->end_date, p_block->i_pts);
 340     } else if (p_block->i_pts < date_Get(&p_sys->end_date)) {
 341         msg_Warn(p_dec, "MIDI message in the past?");
 342         goto drop;
 343     }
 344
 345     if (p_block->i_buffer < 1)
 346         goto drop;
 347
 348     uint8_t event = p_block->p_buffer[0];
 349     uint8_t data1 = (p_block->i_buffer > 1) ? (p_block->p_buffer[1]) : 0;
 350     uint8_t data2 = (p_block->i_buffer > 2) ? (p_block->p_buffer[2]) : 0;
 351
 352     switch (event & 0xF0)
 353     {
 354         case kMidiMessage_NoteOff:
 355         case kMidiMessage_NoteOn:
 356         case kMidiMessage_PolyPressure:
 357         case kMidiMessage_ControlChange:
 358         case kMidiMessage_ProgramChange:
 359         case kMidiMessage_ChannelPressure:
 360         case kMidiMessage_PitchWheel:
 361             MusicDeviceMIDIEvent(p_sys->synthUnit, event, data1, data2, 0);
 362         break;
 363
 364         case kMidiMessage_SysEx:
 365             if (p_block->i_buffer < UINT32_MAX)
 366                 MusicDeviceSysEx(p_sys->synthUnit, p_block->p_buffer, (UInt32)p_block->i_buffer);
 367         break;
 368
 369         default:
 370             msg_Warn(p_dec, "unhandled MIDI event: %x", event & 0xF0);
 371         break;
 372     }
 373
 374     // Calculate frame count
 375     // Simplification of 44100 / 1000000
 376     // TODO: Other samplerates
 377     unsigned frames =
 378        (p_block->i_pts - date_Get(&p_sys->end_date)) * 441 / 10000;
 379
 380     if (frames == 0)
 381         goto drop;
 382
 383     p_out = decoder_NewAudioBuffer(p_dec, frames);
 384     if (p_out == NULL)
 385         goto drop;
 386
 387     p_out->i_pts = date_Get(&p_sys->end_date );
 388     p_out->i_length = date_Increment(&p_sys->end_date, frames)
 389                       - p_out->i_pts;
 390
 391     // Prepare Timestamp for the AudioUnit render call
 392     AudioTimeStamp timestamp = {};
 393     timestamp.mFlags = kAudioTimeStampWordClockTimeValid;
 394     timestamp.mWordClockTime = p_out->i_pts;
 395
 396     // Prepare Buffer for the AudioUnit render call
 397     AudioBufferList bufferList;
 398     bufferList.mNumberBuffers = 1;
 399     bufferList.mBuffers[0].mNumberChannels = 2;
 400     bufferList.mBuffers[0].mDataByteSize = frames * sizeof(Float32) * 2;
 401     bufferList.mBuffers[0].mData = p_out->p_buffer;
 402
 403     status = AudioUnitRender(p_sys->outputUnit,
 404                              NULL,
 405                              &timestamp, 0,
 406                              frames, &bufferList);
 407
 408     if (status != noErr) {
 409         msg_Warn(p_dec, "rendering audio unit failed: %i", status);
 410         block_Release(p_out);
 411         p_out = NULL;
 412     }
 413
 414 drop:
 415     block_Release(p_block);
 416     if (p_out != NULL)
 417         decoder_QueueAudio(p_dec, p_out);
 418     return VLCDEC_SUCCESS;
 419 }