modules/codec/audiotoolbox_midi.c

   1 /*****************************************************************************
   2  * audiotoolbox_midi.c: Software MIDI synthesizer using AudioToolbox
   3  *****************************************************************************
   4  * Copyright (C) 2017 VLC authors and VideoLAN
   5  * $Id$
   6  *
   7  * Authors: Marvin Scholz <epirat07 at gmail dot com>
   8  *
   9  * Based on the fluidsynth module by Rémi Denis-Courmont
  10  *
  11  * This program is free software; you can redistribute it and/or modify it
  12  * under the terms of the GNU Lesser General Public License as published by
  13  * the Free Software Foundation; either version 2.1 of the License, or
  14  * (at your option) any later version.
  15  *
  16  * This program is distributed in the hope that it will be useful,
  17  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  18  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  19  * GNU Lesser General Public License for more details.
  20  *
  21  * You should have received a copy of the GNU Lesser General Public License
  22  * along with this program; if not, write to the Free Software Foundation,
  23  * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  24  *****************************************************************************/
  25
  26 #ifdef HAVE_CONFIG_H
  27 # include "config.h"
  28 #endif
  29
  30 #include <vlc_common.h>
  31 #include <vlc_plugin.h>
  32 #include <vlc_codec.h>
  33 #include <vlc_dialog.h>
  34
  35 #include <CoreFoundation/CoreFoundation.h>
  36 #include <AudioUnit/AudioUnit.h>
  37 #include <AudioToolbox/AudioToolbox.h>
  38
  39 #include <TargetConditionals.h>
  40
  41 #ifndef on_err_goto
  42 #define on_err_goto(errorCode, exceptionLabel)  \
  43 do { if ((errorCode) != noErr) goto exceptionLabel; \
  44 } while ( 0 )
  45 #endif
  46
  47 #define SOUNDFONT_TEXT N_("SoundFont file")
  48 #define SOUNDFONT_LONGTEXT N_( \
  49     "SoundFont file to use for software synthesis." )
  50
  51 static int  Open  (vlc_object_t *);
  52 static void Close (vlc_object_t *);
  53
  54 #define CFG_PREFIX "aumidi-"
  55
  56 vlc_module_begin()
  57     set_description(N_("AudioToolbox MIDI synthesizer"))
  58     set_capability("audio decoder", 100)
  59     set_shortname(N_("AUMIDI"))
  60     set_category(CAT_INPUT)
  61     set_subcategory(SUBCAT_INPUT_ACODEC)
  62     set_callbacks(Open, Close)
  63     add_loadfile(CFG_PREFIX "soundfont", "",
  64                  SOUNDFONT_TEXT, SOUNDFONT_LONGTEXT)
  65 vlc_module_end()
  66
  67
  68 typedef struct
  69 {
  70     AUGraph     graph;
  71     AudioUnit   synthUnit;
  72     AudioUnit   outputUnit;
  73     date_t       end_date;
  74 } decoder_sys_t;
  75
  76 static int  DecodeBlock (decoder_t *p_dec, block_t *p_block);
  77 static void Flush (decoder_t *);
  78
  79 /* MIDI constants */
  80 enum
  81 {
  82     kMidiMessage_NoteOff            = 0x80,
  83     kMidiMessage_NoteOn             = 0x90,
  84     kMidiMessage_PolyPressure       = 0xA0,
  85     kMidiMessage_ControlChange      = 0xB0,
  86     kMidiMessage_ProgramChange      = 0xC0,
  87     kMidiMessage_ChannelPressure    = 0xD0,
  88     kMidiMessage_PitchWheel         = 0xE0,
  89     kMidiMessage_SysEx              = 0xF0,
  90
  91     kMidiMessage_BankMSBControl     = 0,
  92     kMidiMessage_BankLSBControl     = 32,
  93
  94     /* Values for kMidiMessage_ControlChange */
  95     kMidiController_AllSoundOff         = 0x78,
  96     kMidiController_ResetAllControllers = 0x79,
  97     kMidiController_AllNotesOff         = 0x7B
  98 };
  99
 100 /* Helper functions */
 101 static OSStatus AddAppleAUNode(AUGraph graph, OSType type, OSType subtype, AUNode *node)
 102 {
 103     AudioComponentDescription cDesc = {};
 104     cDesc.componentType = type;
 105     cDesc.componentSubType = subtype;
 106     cDesc.componentManufacturer = kAudioUnitManufacturer_Apple;
 107
 108     return AUGraphAddNode(graph, &cDesc, node);
 109 }
 110
 111 static OSStatus CreateAUGraph(AUGraph *outGraph, AudioUnit *outSynth, AudioUnit *outOut)
 112 {
 113     OSStatus res;
 114
 115     // AudioUnit nodes
 116     AUNode synthNode, limiterNode, outNode;
 117
 118     // Create the Graph to which we will add our nodes
 119     on_err_goto(res = NewAUGraph(outGraph), bailout);
 120
 121     // Create/add the MIDI synthesizer node (DLS Synth)
 122 #if TARGET_OS_IPHONE
 123     // On iOS/tvOS use MIDISynth, DLSSynth does not exist there
 124     on_err_goto(res = AddAppleAUNode(*outGraph,
 125                                      kAudioUnitType_MusicDevice,
 126                                      kAudioUnitSubType_MIDISynth,
 127                                      &synthNode), bailout);
 128 #else
 129     // Prefer DLSSynth on macOS, as it has a better default behavior
 130     on_err_goto(res = AddAppleAUNode(*outGraph,
 131                                      kAudioUnitType_MusicDevice,
 132                                      kAudioUnitSubType_DLSSynth,
 133                                      &synthNode), bailout);
 134 #endif
 135
 136     // Create/add the peak limiter node
 137     on_err_goto(res = AddAppleAUNode(*outGraph,
 138                                      kAudioUnitType_Effect,
 139                                      kAudioUnitSubType_PeakLimiter,
 140                                      &limiterNode), bailout);
 141
 142     // Create/add the output node (GenericOutput)
 143     on_err_goto(res = AddAppleAUNode(*outGraph,
 144                                      kAudioUnitType_Output,
 145                                      kAudioUnitSubType_GenericOutput,
 146                                      &outNode), bailout);
 147
 148     // Open the Graph, this opens the units that belong to the graph
 149     // so that we can connect them
 150     on_err_goto(res = AUGraphOpen(*outGraph), bailout);
 151
 152     // Connect the synthesizer node to the limiter
 153     on_err_goto(res = AUGraphConnectNodeInput(*outGraph, synthNode, 0, limiterNode, 0), bailout);
 154     // Connect the limiter node to the output
 155     on_err_goto(res = AUGraphConnectNodeInput(*outGraph, limiterNode, 0, outNode, 0), bailout);
 156
 157     // Get reference to the synthesizer node
 158     on_err_goto(res = AUGraphNodeInfo(*outGraph, synthNode, 0, outSynth), bailout);
 159     // Get reference to the output node
 160     on_err_goto(res = AUGraphNodeInfo(*outGraph, outNode, 0, outOut), bailout);
 161
 162 bailout:
 163     return res;
 164 }
 165
 166 static int SetSoundfont(decoder_t *p_dec, AudioUnit synthUnit, const char *sfPath) {
 167     if (!sfPath) {
 168         msg_Dbg(p_dec, "using default soundfont");
 169         return VLC_SUCCESS;
 170     }
 171
 172     msg_Dbg(p_dec, "using custom soundfont: '%s'", sfPath);
 173     CFURLRef url = CFURLCreateFromFileSystemRepresentation(kCFAllocatorDefault,
 174                                                            (const UInt8 *)sfPath,
 175                                                            strlen(sfPath), false);
 176     if (unlikely(url == NULL))
 177         return VLC_ENOMEM;
 178
 179     OSStatus status = AudioUnitSetProperty(synthUnit,
 180                                            kMusicDeviceProperty_SoundBankURL,
 181                                            kAudioUnitScope_Global, 0,
 182                                            &url, sizeof(url));
 183     CFRelease(url);
 184
 185     if (status != noErr) {
 186         msg_Err(p_dec, "failed setting custom SoundFont for MIDI synthesis (%i)", status);
 187         return VLC_EGENERIC;
 188     }
 189     return VLC_SUCCESS;
 190 }
 191
 192 static int Open(vlc_object_t *p_this)
 193 {
 194     decoder_t *p_dec = (decoder_t *)p_this;
 195     OSStatus status = noErr;
 196     int ret = VLC_SUCCESS;
 197
 198     if (p_dec->fmt_in.i_codec != VLC_CODEC_MIDI)
 199         return VLC_EGENERIC;
 200
 201     decoder_sys_t *p_sys = malloc(sizeof (*p_sys));
 202     if (unlikely(p_sys == NULL))
 203         return VLC_ENOMEM;
 204
 205     p_sys->graph = NULL;
 206     status = CreateAUGraph(&p_sys->graph, &p_sys->synthUnit, &p_sys->outputUnit);
 207     if (unlikely(status != noErr)) {
 208         msg_Err(p_dec, "failed to create audiograph (%i)", status);
 209         ret = VLC_EGENERIC;
 210         goto bailout;
 211     }
 212
 213     // Set custom soundfont
 214     char *sfPath = var_InheritString(p_dec, CFG_PREFIX "soundfont");
 215     ret = SetSoundfont(p_dec, p_sys->synthUnit, sfPath);
 216     free(sfPath);
 217     if (unlikely(ret != VLC_SUCCESS))
 218         goto bailout;
 219
 220     // Set VLC output audio format info
 221     p_dec->fmt_out.i_codec = VLC_CODEC_FL32;
 222     p_dec->fmt_out.audio.i_bitspersample = 32;
 223     p_dec->fmt_out.audio.i_rate = 44100;
 224     p_dec->fmt_out.audio.i_channels = 2;
 225     p_dec->fmt_out.audio.i_physical_channels = AOUT_CHAN_LEFT | AOUT_CHAN_RIGHT;
 226
 227     if (decoder_UpdateAudioFormat(p_dec) < 0) {
 228         ret = VLC_EGENERIC;
 229         goto bailout;
 230     }
 231
 232     // Prepare AudioUnit output audio format info
 233     AudioStreamBasicDescription ASBD = {};
 234     unsigned bytesPerSample = sizeof(Float32);
 235     ASBD.mFormatID = kAudioFormatLinearPCM;
 236     ASBD.mFormatFlags = kAudioFormatFlagIsFloat | kAudioFormatFlagIsPacked;
 237     ASBD.mSampleRate = 44100;
 238     ASBD.mFramesPerPacket = 1;
 239     ASBD.mChannelsPerFrame = 2;
 240     ASBD.mBytesPerFrame = bytesPerSample * ASBD.mChannelsPerFrame;
 241     ASBD.mBytesPerPacket = ASBD.mBytesPerFrame * ASBD.mFramesPerPacket;
 242     ASBD.mBitsPerChannel = 8 * bytesPerSample;
 243
 244     // Set AudioUnit format
 245     status = AudioUnitSetProperty(p_sys->outputUnit,
 246                                   kAudioUnitProperty_StreamFormat,
 247                                   kAudioUnitScope_Output, 0, &ASBD,
 248                                   sizeof(AudioStreamBasicDescription));
 249     if (unlikely(status != noErr)) {
 250         msg_Err(p_dec, "failed setting output format for output unit (%i)", status);
 251         ret = VLC_EGENERIC;
 252         goto bailout;
 253     }
 254
 255     // Prepare the AU
 256     status = AUGraphInitialize (p_sys->graph);
 257     if (unlikely(status != noErr)) {
 258         if (status == kAudioUnitErr_InvalidFile)
 259             msg_Err(p_dec, "failed initializing audiograph: invalid soundfont file");
 260         else
 261             msg_Err(p_dec, "failed initializing audiograph (%i)", status);
 262         ret = VLC_EGENERIC;
 263         goto bailout;
 264     }
 265
 266     // Prepare MIDI soundbank
 267     MusicDeviceMIDIEvent(p_sys->synthUnit,
 268                          kMidiMessage_ControlChange,
 269                          kMidiMessage_BankMSBControl, 0, 0);
 270
 271     // Start the AU
 272     status = AUGraphStart(p_sys->graph);
 273     if (unlikely(status != noErr)) {
 274         msg_Err(p_dec, "failed starting audiograph (%i)", status);
 275         ret = VLC_EGENERIC;
 276         goto bailout;
 277     }
 278
 279     // Initialize date (for PTS)
 280     date_Init(&p_sys->end_date, p_dec->fmt_out.audio.i_rate, 1);
 281
 282     p_dec->p_sys = p_sys;
 283     p_dec->pf_decode = DecodeBlock;
 284     p_dec->pf_flush  = Flush;
 285
 286 bailout:
 287     // Cleanup if error occured
 288     if (ret != VLC_SUCCESS) {
 289         if (p_sys->graph)
 290             DisposeAUGraph(p_sys->graph);
 291         free(p_sys);
 292     }
 293     return ret;
 294 }
 295
 296
 297 static void Close (vlc_object_t *p_this)
 298 {
 299     decoder_sys_t *p_sys = ((decoder_t *)p_this)->p_sys;
 300
 301     if (p_sys->graph) {
 302         AUGraphStop(p_sys->graph);
 303         DisposeAUGraph(p_sys->graph);
 304     }
 305     free(p_sys);
 306 }
 307
 308 static void Flush (decoder_t *p_dec)
 309 {
 310     decoder_sys_t *p_sys = p_dec->p_sys;
 311
 312     date_Set(&p_sys->end_date, VLC_TICK_INVALID);
 313
 314     // Turn all sound on all channels off
 315     // else 'old' notes could still be playing
 316     for (unsigned channel = 0; channel < 16; channel++) {
 317         MusicDeviceMIDIEvent(p_sys->synthUnit, kMidiMessage_ControlChange | channel, kMidiController_AllSoundOff, 0, 0);
 318     }
 319 }
 320
 321 static int DecodeBlock (decoder_t *p_dec, block_t *p_block)
 322 {
 323     decoder_sys_t *p_sys = p_dec->p_sys;
 324     block_t *p_out = NULL;
 325     OSStatus status = noErr;
 326
 327     if (p_block == NULL) /* No Drain */
 328         return VLCDEC_SUCCESS;
 329
 330     if (p_block->i_flags & (BLOCK_FLAG_DISCONTINUITY|BLOCK_FLAG_CORRUPTED)) {
 331         Flush(p_dec);
 332         if (p_block->i_flags & BLOCK_FLAG_CORRUPTED) {
 333             block_Release(p_block);
 334             return VLCDEC_SUCCESS;
 335         }
 336     }
 337
 338     if ( p_block->i_pts != VLC_TICK_INVALID &&
 339          date_Get(&p_sys->end_date) == VLC_TICK_INVALID ) {
 340         date_Set(&p_sys->end_date, p_block->i_pts);
 341     } else if (p_block->i_pts < date_Get(&p_sys->end_date)) {
 342         msg_Warn(p_dec, "MIDI message in the past?");
 343         goto drop;
 344     }
 345
 346     if (p_block->i_buffer < 1)
 347         goto drop;
 348
 349     uint8_t event = p_block->p_buffer[0];
 350     uint8_t data1 = (p_block->i_buffer > 1) ? (p_block->p_buffer[1]) : 0;
 351     uint8_t data2 = (p_block->i_buffer > 2) ? (p_block->p_buffer[2]) : 0;
 352
 353     switch (event & 0xF0)
 354     {
 355         case kMidiMessage_NoteOff:
 356         case kMidiMessage_NoteOn:
 357         case kMidiMessage_PolyPressure:
 358         case kMidiMessage_ControlChange:
 359         case kMidiMessage_ProgramChange:
 360         case kMidiMessage_ChannelPressure:
 361         case kMidiMessage_PitchWheel:
 362             MusicDeviceMIDIEvent(p_sys->synthUnit, event, data1, data2, 0);
 363         break;
 364
 365         case kMidiMessage_SysEx:
 366             if (p_block->i_buffer < UINT32_MAX)
 367                 MusicDeviceSysEx(p_sys->synthUnit, p_block->p_buffer, (UInt32)p_block->i_buffer);
 368         break;
 369
 370         default:
 371             msg_Warn(p_dec, "unhandled MIDI event: %x", event & 0xF0);
 372         break;
 373     }
 374
 375     // Calculate frame count
 376     // Simplification of 44100 / 1000000
 377     // TODO: Other samplerates
 378     unsigned frames =
 379        (p_block->i_pts - date_Get(&p_sys->end_date)) * 441 / 10000;
 380
 381     if (frames == 0)
 382         goto drop;
 383
 384     p_out = decoder_NewAudioBuffer(p_dec, frames);
 385     if (p_out == NULL)
 386         goto drop;
 387
 388     p_out->i_pts = date_Get(&p_sys->end_date );
 389     p_out->i_length = date_Increment(&p_sys->end_date, frames)
 390                       - p_out->i_pts;
 391
 392     // Prepare Timestamp for the AudioUnit render call
 393     AudioTimeStamp timestamp = {};
 394     timestamp.mFlags = kAudioTimeStampWordClockTimeValid;
 395     timestamp.mWordClockTime = p_out->i_pts;
 396
 397     // Prepare Buffer for the AudioUnit render call
 398     AudioBufferList bufferList;
 399     bufferList.mNumberBuffers = 1;
 400     bufferList.mBuffers[0].mNumberChannels = 2;
 401     bufferList.mBuffers[0].mDataByteSize = frames * sizeof(Float32) * 2;
 402     bufferList.mBuffers[0].mData = p_out->p_buffer;
 403
 404     status = AudioUnitRender(p_sys->outputUnit,
 405                              NULL,
 406                              &timestamp, 0,
 407                              frames, &bufferList);
 408
 409     if (status != noErr) {
 410         msg_Warn(p_dec, "rendering audio unit failed: %i", status);
 411         block_Release(p_out);
 412         p_out = NULL;
 413     }
 414
 415 drop:
 416     block_Release(p_block);
 417     if (p_out != NULL)
 418         decoder_QueueAudio(p_dec, p_out);
 419     return VLCDEC_SUCCESS;
 420 }