1 /*****************************************************************************
2 * audiotoolbox_midi.c: Software MIDI synthesizer using AudioToolbox
3 *****************************************************************************
4 * Copyright (C) 2017 VLC authors and VideoLAN
7 * Authors: Marvin Scholz <epirat07 at gmail dot com>
9 * Based on the fluidsynth module by RĂ©mi Denis-Courmont
11 * This program is free software; you can redistribute it and/or modify it
12 * under the terms of the GNU Lesser General Public License as published by
13 * the Free Software Foundation; either version 2.1 of the License, or
14 * (at your option) any later version.
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU Lesser General Public License for more details.
21 * You should have received a copy of the GNU Lesser General Public License
22 * along with this program; if not, write to the Free Software Foundation,
23 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
24 *****************************************************************************/
30 #include <vlc_common.h>
31 #include <vlc_plugin.h>
32 #include <vlc_codec.h>
33 #include <vlc_dialog.h>
35 #include <CoreFoundation/CoreFoundation.h>
36 #include <AudioUnit/AudioUnit.h>
37 #include <AudioToolbox/AudioToolbox.h>
39 #include <TargetConditionals.h>
42 #define on_err_goto(errorCode, exceptionLabel) \
43 do { if ((errorCode) != noErr) goto exceptionLabel; \
47 #define SOUNDFONT_TEXT N_("SoundFont file")
48 #define SOUNDFONT_LONGTEXT N_( \
49 "SoundFont file to use for software synthesis." )
51 static int Open (vlc_object_t
*);
52 static void Close (vlc_object_t
*);
54 #define CFG_PREFIX "aumidi-"
57 set_description(N_("AudioToolbox MIDI synthesizer"))
58 set_capability("audio decoder", 100)
59 set_shortname(N_("AUMIDI"))
60 set_category(CAT_INPUT
)
61 set_subcategory(SUBCAT_INPUT_ACODEC
)
62 set_callbacks(Open
, Close
)
63 add_loadfile("soundfont", "",
64 SOUNDFONT_TEXT
, SOUNDFONT_LONGTEXT
, false)
76 static int DecodeBlock (decoder_t
*p_dec
, block_t
*p_block
);
77 static void Flush (decoder_t
*);
82 kMidiMessage_NoteOff
= 0x80,
83 kMidiMessage_NoteOn
= 0x90,
84 kMidiMessage_PolyPressure
= 0xA0,
85 kMidiMessage_ControlChange
= 0xB0,
86 kMidiMessage_ProgramChange
= 0xC0,
87 kMidiMessage_ChannelPressure
= 0xD0,
88 kMidiMessage_PitchWheel
= 0xE0,
89 kMidiMessage_SysEx
= 0xF0,
91 kMidiMessage_BankMSBControl
= 0,
92 kMidiMessage_BankLSBControl
= 32,
94 /* Values for kMidiMessage_ControlChange */
95 kMidiController_AllSoundOff
= 0x78,
96 kMidiController_ResetAllControllers
= 0x79,
97 kMidiController_AllNotesOff
= 0x7B
100 /* Helper functions */
101 static OSStatus
AddAppleAUNode(AUGraph graph
, OSType type
, OSType subtype
, AUNode
*node
)
103 AudioComponentDescription cDesc
= {};
104 cDesc
.componentType
= type
;
105 cDesc
.componentSubType
= subtype
;
106 cDesc
.componentManufacturer
= kAudioUnitManufacturer_Apple
;
108 return AUGraphAddNode(graph
, &cDesc
, node
);
111 static OSStatus
CreateAUGraph(AUGraph
*outGraph
, AudioUnit
*outSynth
, AudioUnit
*outOut
)
116 AUNode synthNode
, limiterNode
, outNode
;
118 // Create the Graph to which we will add our nodes
119 on_err_goto(res
= NewAUGraph(outGraph
), bailout
);
121 // Create/add the MIDI synthesizer node (DLS Synth)
123 // On iOS/tvOS use MIDISynth, DLSSynth does not exist there
124 on_err_goto(res
= AddAppleAUNode(*outGraph
,
125 kAudioUnitType_MusicDevice
,
126 kAudioUnitSubType_MIDISynth
,
127 &synthNode
), bailout
);
129 // Prefer DLSSynth on macOS, as it has a better default behavior
130 on_err_goto(res
= AddAppleAUNode(*outGraph
,
131 kAudioUnitType_MusicDevice
,
132 kAudioUnitSubType_DLSSynth
,
133 &synthNode
), bailout
);
136 // Create/add the peak limiter node
137 on_err_goto(res
= AddAppleAUNode(*outGraph
,
138 kAudioUnitType_Effect
,
139 kAudioUnitSubType_PeakLimiter
,
140 &limiterNode
), bailout
);
142 // Create/add the output node (GenericOutput)
143 on_err_goto(res
= AddAppleAUNode(*outGraph
,
144 kAudioUnitType_Output
,
145 kAudioUnitSubType_GenericOutput
,
148 // Open the Graph, this opens the units that belong to the graph
149 // so that we can connect them
150 on_err_goto(res
= AUGraphOpen(*outGraph
), bailout
);
152 // Connect the synthesizer node to the limiter
153 on_err_goto(res
= AUGraphConnectNodeInput(*outGraph
, synthNode
, 0, limiterNode
, 0), bailout
);
154 // Connect the limiter node to the output
155 on_err_goto(res
= AUGraphConnectNodeInput(*outGraph
, limiterNode
, 0, outNode
, 0), bailout
);
157 // Get reference to the synthesizer node
158 on_err_goto(res
= AUGraphNodeInfo(*outGraph
, synthNode
, 0, outSynth
), bailout
);
159 // Get reference to the output node
160 on_err_goto(res
= AUGraphNodeInfo(*outGraph
, outNode
, 0, outOut
), bailout
);
166 static int SetSoundfont(decoder_t
*p_dec
, AudioUnit synthUnit
, const char *sfPath
) {
168 msg_Dbg(p_dec
, "using default soundfont");
172 msg_Dbg(p_dec
, "using custom soundfont: '%s'", sfPath
);
173 CFURLRef url
= CFURLCreateFromFileSystemRepresentation(kCFAllocatorDefault
,
174 (const UInt8
*)sfPath
,
175 strlen(sfPath
), false);
176 if (unlikely(url
== NULL
))
179 OSStatus status
= AudioUnitSetProperty(synthUnit
,
180 kMusicDeviceProperty_SoundBankURL
,
181 kAudioUnitScope_Global
, 0,
185 if (status
!= noErr
) {
186 msg_Err(p_dec
, "failed setting custom SoundFont for MIDI synthesis (%i)", status
);
192 static int Open(vlc_object_t
*p_this
)
194 decoder_t
*p_dec
= (decoder_t
*)p_this
;
195 OSStatus status
= noErr
;
196 int ret
= VLC_SUCCESS
;
198 if (p_dec
->fmt_in
.i_codec
!= VLC_CODEC_MIDI
)
201 decoder_sys_t
*p_sys
= malloc(sizeof (*p_sys
));
202 if (unlikely(p_sys
== NULL
))
206 status
= CreateAUGraph(&p_sys
->graph
, &p_sys
->synthUnit
, &p_sys
->outputUnit
);
207 if (unlikely(status
!= noErr
)) {
208 msg_Err(p_dec
, "failed to create audiograph (%i)", status
);
213 // Set custom soundfont
214 char *sfPath
= var_InheritString(p_dec
, CFG_PREFIX
"soundfont");
215 ret
= SetSoundfont(p_dec
, p_sys
->synthUnit
, sfPath
);
217 if (unlikely(ret
!= VLC_SUCCESS
))
220 // Set VLC output audio format info
221 p_dec
->fmt_out
.i_codec
= VLC_CODEC_FL32
;
222 p_dec
->fmt_out
.audio
.i_bitspersample
= 32;
223 p_dec
->fmt_out
.audio
.i_rate
= 44100;
224 p_dec
->fmt_out
.audio
.i_channels
= 2;
225 p_dec
->fmt_out
.audio
.i_physical_channels
= AOUT_CHAN_LEFT
| AOUT_CHAN_RIGHT
;
227 if (decoder_UpdateAudioFormat(p_dec
) < 0) {
232 // Prepare AudioUnit output audio format info
233 AudioStreamBasicDescription ASBD
= {};
234 unsigned bytesPerSample
= sizeof(Float32
);
235 ASBD
.mFormatID
= kAudioFormatLinearPCM
;
236 ASBD
.mFormatFlags
= kAudioFormatFlagIsFloat
| kAudioFormatFlagIsPacked
;
237 ASBD
.mSampleRate
= 44100;
238 ASBD
.mFramesPerPacket
= 1;
239 ASBD
.mChannelsPerFrame
= 2;
240 ASBD
.mBytesPerFrame
= bytesPerSample
* ASBD
.mChannelsPerFrame
;
241 ASBD
.mBytesPerPacket
= ASBD
.mBytesPerFrame
* ASBD
.mFramesPerPacket
;
242 ASBD
.mBitsPerChannel
= 8 * bytesPerSample
;
244 // Set AudioUnit format
245 status
= AudioUnitSetProperty(p_sys
->outputUnit
,
246 kAudioUnitProperty_StreamFormat
,
247 kAudioUnitScope_Output
, 0, &ASBD
,
248 sizeof(AudioStreamBasicDescription
));
249 if (unlikely(status
!= noErr
)) {
250 msg_Err(p_dec
, "failed setting output format for output unit (%i)", status
);
256 status
= AUGraphInitialize (p_sys
->graph
);
257 if (unlikely(status
!= noErr
)) {
258 if (status
== kAudioUnitErr_InvalidFile
)
259 msg_Err(p_dec
, "failed initializing audiograph: invalid soundfont file");
261 msg_Err(p_dec
, "failed initializing audiograph (%i)", status
);
266 // Prepare MIDI soundbank
267 MusicDeviceMIDIEvent(p_sys
->synthUnit
,
268 kMidiMessage_ControlChange
,
269 kMidiMessage_BankMSBControl
, 0, 0);
272 status
= AUGraphStart(p_sys
->graph
);
273 if (unlikely(status
!= noErr
)) {
274 msg_Err(p_dec
, "failed starting audiograph (%i)", status
);
279 // Initialize date (for PTS)
280 date_Init(&p_sys
->end_date
, p_dec
->fmt_out
.audio
.i_rate
, 1);
281 date_Set(&p_sys
->end_date
, 0);
283 p_dec
->p_sys
= p_sys
;
284 p_dec
->pf_decode
= DecodeBlock
;
285 p_dec
->pf_flush
= Flush
;
288 // Cleanup if error occured
289 if (ret
!= VLC_SUCCESS
) {
291 DisposeAUGraph(p_sys
->graph
);
298 static void Close (vlc_object_t
*p_this
)
300 decoder_sys_t
*p_sys
= ((decoder_t
*)p_this
)->p_sys
;
303 AUGraphStop(p_sys
->graph
);
304 DisposeAUGraph(p_sys
->graph
);
309 static void Flush (decoder_t
*p_dec
)
311 decoder_sys_t
*p_sys
= p_dec
->p_sys
;
313 date_Set(&p_sys
->end_date
, VLC_TS_INVALID
);
315 // Turn all sound on all channels off
316 // else 'old' notes could still be playing
317 for (unsigned channel
= 0; channel
< 16; channel
++) {
318 MusicDeviceMIDIEvent(p_sys
->synthUnit
, kMidiMessage_ControlChange
| channel
, kMidiController_AllSoundOff
, 0, 0);
322 static int DecodeBlock (decoder_t
*p_dec
, block_t
*p_block
)
324 decoder_sys_t
*p_sys
= p_dec
->p_sys
;
325 block_t
*p_out
= NULL
;
326 OSStatus status
= noErr
;
328 if (p_block
== NULL
) /* No Drain */
329 return VLCDEC_SUCCESS
;
331 if (p_block
->i_flags
& (BLOCK_FLAG_DISCONTINUITY
|BLOCK_FLAG_CORRUPTED
)) {
333 if (p_block
->i_flags
& BLOCK_FLAG_CORRUPTED
) {
334 block_Release(p_block
);
335 return VLCDEC_SUCCESS
;
339 if (p_block
->i_pts
> VLC_TS_INVALID
&& !date_Get(&p_sys
->end_date
)) {
340 date_Set(&p_sys
->end_date
, p_block
->i_pts
);
341 } else if (p_block
->i_pts
< date_Get(&p_sys
->end_date
)) {
342 msg_Warn(p_dec
, "MIDI message in the past?");
346 if (p_block
->i_buffer
< 1)
349 uint8_t event
= p_block
->p_buffer
[0];
350 uint8_t data1
= (p_block
->i_buffer
> 1) ? (p_block
->p_buffer
[1]) : 0;
351 uint8_t data2
= (p_block
->i_buffer
> 2) ? (p_block
->p_buffer
[2]) : 0;
353 switch (event
& 0xF0)
355 case kMidiMessage_NoteOff
:
356 case kMidiMessage_NoteOn
:
357 case kMidiMessage_PolyPressure
:
358 case kMidiMessage_ControlChange
:
359 case kMidiMessage_ProgramChange
:
360 case kMidiMessage_ChannelPressure
:
361 case kMidiMessage_PitchWheel
:
362 MusicDeviceMIDIEvent(p_sys
->synthUnit
, event
, data1
, data2
, 0);
365 case kMidiMessage_SysEx
:
366 if (p_block
->i_buffer
< UINT32_MAX
)
367 MusicDeviceSysEx(p_sys
->synthUnit
, p_block
->p_buffer
, (UInt32
)p_block
->i_buffer
);
371 msg_Warn(p_dec
, "unhandled MIDI event: %x", event
& 0xF0);
375 // Calculate frame count
376 // Simplification of 44100 / 1000000
377 // TODO: Other samplerates
379 (p_block
->i_pts
- date_Get(&p_sys
->end_date
)) * 441 / 10000;
384 p_out
= decoder_NewAudioBuffer(p_dec
, frames
);
388 p_out
->i_pts
= date_Get(&p_sys
->end_date
);
389 p_out
->i_length
= date_Increment(&p_sys
->end_date
, frames
)
392 // Prepare Timestamp for the AudioUnit render call
393 AudioTimeStamp timestamp
= {};
394 timestamp
.mFlags
= kAudioTimeStampWordClockTimeValid
;
395 timestamp
.mWordClockTime
= p_out
->i_pts
;
397 // Prepare Buffer for the AudioUnit render call
398 AudioBufferList bufferList
;
399 bufferList
.mNumberBuffers
= 1;
400 bufferList
.mBuffers
[0].mNumberChannels
= 2;
401 bufferList
.mBuffers
[0].mDataByteSize
= frames
* sizeof(Float32
) * 2;
402 bufferList
.mBuffers
[0].mData
= p_out
->p_buffer
;
404 status
= AudioUnitRender(p_sys
->outputUnit
,
407 frames
, &bufferList
);
409 if (status
!= noErr
) {
410 msg_Warn(p_dec
, "rendering audio unit failed: %i", status
);
411 block_Release(p_out
);
416 block_Release(p_block
);
418 decoder_QueueAudio(p_dec
, p_out
);
419 return VLCDEC_SUCCESS
;