1 /*****************************************************************************
2 * videotoolbox.m: Video Toolbox decoder
3 *****************************************************************************
4 * Copyright © 2014-2015 VideoLabs SAS
6 * Authors: Felix Paul Kühne <fkuehne # videolan.org>
8 * This program is free software; you can redistribute it and/or modify it
9 * under the terms of the GNU Lesser General Public License as published by
10 * the Free Software Foundation; either version 2.1 of the License, or
11 * (at your option) any later version.
13 * This program is distributed in the hope that it will be useful,
14 * but WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
16 * GNU Lesser General Public License for more details.
18 * You should have received a copy of the GNU Lesser General Public License
19 * along with this program; if not, write to the Free Software Foundation,
20 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
21 *****************************************************************************/
29 #import <vlc_common.h>
30 #import <vlc_plugin.h>
32 #import "hxxx_helper.h"
36 #import "../packetizer/h264_nal.h"
37 #import "../packetizer/h264_slice.h"
38 #import "../packetizer/hxxx_nal.h"
39 #import "../packetizer/hxxx_sei.h"
41 #import <VideoToolbox/VideoToolbox.h>
42 #import <VideoToolbox/VTErrors.h>
44 #import <Foundation/Foundation.h>
45 #import <TargetConditionals.h>
48 #import <sys/sysctl.h>
49 #import <mach/machine.h>
51 #define ALIGN_16( x ) ( ( ( x ) + 15 ) / 16 * 16 )
52 #define VT_RESTART_MAX 1
55 #import <UIKit/UIKit.h>
57 /* support iOS SDKs < v9.1 */
58 #ifndef CPUFAMILY_ARM_TWISTER
59 #define CPUFAMILY_ARM_TWISTER 0x92fb37c8
64 // Define stuff for older SDKs
65 #if (TARGET_OS_OSX && MAC_OS_X_VERSION_MAX_ALLOWED < 101100) || \
66 (TARGET_OS_IPHONE && __IPHONE_OS_VERSION_MAX_ALLOWED < 90000) || \
67 (TARGET_OS_TV && __TV_OS_VERSION_MAX_ALLOWED < 90000)
68 enum { kCMVideoCodecType_HEVC = 'hvc1' };
71 #if (!TARGET_OS_OSX || MAC_OS_X_VERSION_MAX_ALLOWED < 1090)
72 const CFStringRef kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder = CFSTR("EnableHardwareAcceleratedVideoDecoder");
73 const CFStringRef kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder = CFSTR("RequireHardwareAcceleratedVideoDecoder");
76 #pragma mark - module descriptor
78 static int OpenDecoder(vlc_object_t *);
79 static void CloseDecoder(vlc_object_t *);
81 #define VT_ENABLE_TEXT N_("Enable hardware acceleration")
82 #define VT_REQUIRE_HW_DEC N_("Use Hardware decoders only")
83 #define VT_FORCE_CVPX_CHROMA "Force the VT decoder CVPX chroma"
84 #define VT_FORCE_CVPX_CHROMA_LONG "Values can be 'BGRA', 'y420', '420f', '420v', '2vuy'. \
85 By Default, the best chroma is choosen by the VT decoder."
88 set_category(CAT_INPUT)
89 set_subcategory(SUBCAT_INPUT_VCODEC)
90 set_description(N_("VideoToolbox video decoder"))
91 set_capability("video decoder",800)
92 set_callbacks(OpenDecoder, CloseDecoder)
94 add_obsolete_bool("videotoolbox-temporal-deinterlacing")
95 add_bool("videotoolbox", true, VT_ENABLE_TEXT, NULL, false)
96 add_bool("videotoolbox-hw-decoder-only", true, VT_REQUIRE_HW_DEC, VT_REQUIRE_HW_DEC, false)
97 add_string("videotoolbox-cvpx-chroma", "", VT_FORCE_CVPX_CHROMA, VT_FORCE_CVPX_CHROMA_LONG, true);
100 #pragma mark - local prototypes
102 enum vtsession_status
105 VTSESSION_STATUS_RESTART,
106 VTSESSION_STATUS_RESTART_CHROMA,
107 VTSESSION_STATUS_ABORT,
110 static int ConfigureVout(decoder_t *);
111 static CFMutableDictionaryRef ESDSExtradataInfoCreate(decoder_t *, uint8_t *, uint32_t);
112 static CFMutableDictionaryRef ExtradataInfoCreate(CFStringRef, void *, size_t);
113 static CFMutableDictionaryRef CreateSessionDescriptionFormat(decoder_t *, unsigned, unsigned);
114 static int HandleVTStatus(decoder_t *, OSStatus, enum vtsession_status *);
115 static int DecodeBlock(decoder_t *, block_t *);
116 static void RequestFlush(decoder_t *);
117 static void Drain(decoder_t *p_dec, bool flush);
118 static void DecoderCallback(void *, void *, OSStatus, VTDecodeInfoFlags,
119 CVPixelBufferRef, CMTime, CMTime);
120 static BOOL deviceSupportsHEVC();
121 static BOOL deviceSupportsAdvancedProfiles();
122 static BOOL deviceSupportsAdvancedLevels();
124 typedef struct frame_info_t frame_info_t;
128 picture_t *p_picture;
136 bool b_top_field_first;
139 frame_info_t *p_next;
142 #pragma mark - decoder structure
144 #define H264_MAX_DPB 16
145 #define VT_MAX_SEI_COUNT 16
147 typedef struct decoder_sys_t
149 CMVideoCodecType codec;
150 struct hxxx_helper hh;
152 /* Codec specific callbacks */
153 bool (*pf_codec_init)(decoder_t *);
154 void (*pf_codec_clean)(decoder_t *);
155 bool (*pf_codec_supported)(decoder_t *);
156 bool (*pf_late_start)(decoder_t *);
157 block_t* (*pf_process_block)(decoder_t *,
159 bool (*pf_need_restart)(decoder_t *,
160 VTDecompressionSessionRef);
161 bool (*pf_configure_vout)(decoder_t *);
162 CFMutableDictionaryRef (*pf_get_extradata)(decoder_t *);
163 bool (*pf_fill_reorder_info)(decoder_t *, const block_t *,
165 /* !Codec specific callbacks */
169 bool b_vt_need_keyframe;
170 VTDecompressionSessionRef session;
171 CMVideoFormatDescriptionRef videoFormatDescription;
174 frame_info_t *p_pic_reorder;
175 uint8_t i_pic_reorder;
176 uint8_t i_pic_reorder_max;
177 bool b_invalid_pic_reorder_max;
178 bool b_poc_based_reorder;
180 bool b_format_propagated;
182 enum vtsession_status vtsession_status;
183 unsigned i_restart_count;
186 bool b_cvpx_format_forced;
188 h264_poc_context_t h264_pocctx;
189 hevc_poc_ctx_t hevc_pocctx;
193 struct pic_holder *pic_holder;
201 uint8_t nb_field_out;
202 uint8_t field_reorder_max;
205 static void pic_holder_update_reorder_max(struct pic_holder *, uint8_t, uint8_t);
207 #pragma mark - start & stop
211 static void HXXXGetBestChroma(decoder_t *p_dec)
213 decoder_sys_t *p_sys = p_dec->p_sys;
215 if (p_sys->i_cvpx_format != 0 || p_sys->b_cvpx_format_forced)
218 uint8_t i_chroma_format, i_depth_luma, i_depth_chroma;
219 if (hxxx_helper_get_chroma_chroma(&p_sys->hh, &i_chroma_format, &i_depth_luma,
220 &i_depth_chroma) != VLC_SUCCESS)
223 if (i_chroma_format == 1 /* YUV 4:2:0 */)
225 if (i_depth_luma == 8 && i_depth_chroma == 8)
226 p_sys->i_cvpx_format = kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange;
227 #if !TARGET_OS_IPHONE
228 /* Not for iOS since there is no 10bits textures with the old iOS
229 * openGLES version, and therefore no P010 shaders */
230 else if (i_depth_luma == 10 && i_depth_chroma == 10 && deviceSupportsHEVC())
231 p_sys->i_cvpx_format = 'x420'; /* kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange */
236 static void GetxPSH264(uint8_t i_pps_id, void *priv,
237 const h264_sequence_parameter_set_t **pp_sps,
238 const h264_picture_parameter_set_t **pp_pps)
240 decoder_sys_t *p_sys = priv;
242 *pp_pps = p_sys->hh.h264.pps_list[i_pps_id].h264_pps;
246 *pp_sps = p_sys->hh.h264.sps_list[(*pp_pps)->i_sps_id].h264_sps;
249 struct sei_callback_h264_s
251 uint8_t i_pic_struct;
252 const h264_sequence_parameter_set_t *p_sps;
255 static bool ParseH264SEI(const hxxx_sei_data_t *p_sei_data, void *priv)
257 if(p_sei_data->i_type == HXXX_SEI_PIC_TIMING)
259 struct sei_callback_h264_s *s = priv;
260 if(s->p_sps && s->p_sps->vui.b_valid)
262 if(s->p_sps->vui.b_hrd_parameters_present_flag)
264 bs_read(p_sei_data->p_bs, s->p_sps->vui.i_cpb_removal_delay_length_minus1 + 1);
265 bs_read(p_sei_data->p_bs, s->p_sps->vui.i_dpb_output_delay_length_minus1 + 1);
268 if(s->p_sps->vui.b_pic_struct_present_flag)
269 s->i_pic_struct = bs_read( p_sei_data->p_bs, 4);
277 static bool FillReorderInfoH264(decoder_t *p_dec, const block_t *p_block,
278 frame_info_t *p_info)
280 decoder_sys_t *p_sys = p_dec->p_sys;
281 hxxx_iterator_ctx_t itctx;
282 hxxx_iterator_init(&itctx, p_block->p_buffer, p_block->i_buffer,
283 p_sys->hh.i_nal_length_size);
285 const uint8_t *p_nal; size_t i_nal;
288 const uint8_t *p_nal;
290 } sei_array[VT_MAX_SEI_COUNT];
291 size_t i_sei_count = 0;
292 while(hxxx_iterate_next(&itctx, &p_nal, &i_nal))
297 const enum h264_nal_unit_type_e i_nal_type = p_nal[0] & 0x1F;
299 if (i_nal_type <= H264_NAL_SLICE_IDR && i_nal_type != H264_NAL_UNKNOWN)
302 if(!h264_decode_slice(p_nal, i_nal, GetxPSH264, p_sys, &slice))
305 const h264_sequence_parameter_set_t *p_sps;
306 const h264_picture_parameter_set_t *p_pps;
307 GetxPSH264(slice.i_pic_parameter_set_id, p_sys, &p_sps, &p_pps);
311 h264_compute_poc(p_sps, &slice, &p_sys->h264_pocctx,
312 &p_info->i_poc, &p_info->i_foc, &bFOC);
314 p_info->b_keyframe = slice.type == H264_SLICE_TYPE_I;
315 p_info->b_flush = (slice.type == H264_SLICE_TYPE_I) || slice.has_mmco5;
316 p_info->b_field = slice.i_field_pic_flag;
317 p_info->b_progressive = !p_sps->mb_adaptive_frame_field_flag &&
318 !slice.i_field_pic_flag;
320 struct sei_callback_h264_s sei;
322 sei.i_pic_struct = UINT8_MAX;
324 for(size_t i=0; i<i_sei_count; i++)
325 HxxxParseSEI(sei_array[i].p_nal, sei_array[i].i_nal, 1,
328 p_info->i_num_ts = h264_get_num_ts(p_sps, &slice, sei.i_pic_struct,
329 p_info->i_foc, bFOC);
331 if(!p_info->b_progressive)
332 p_info->b_top_field_first = (sei.i_pic_struct % 2 == 1);
334 /* Set frame rate for timings in case of missing rate */
335 if( (!p_dec->fmt_in.video.i_frame_rate_base ||
336 !p_dec->fmt_in.video.i_frame_rate) &&
337 p_sps->vui.i_time_scale && p_sps->vui.i_num_units_in_tick )
339 date_Change( &p_sys->pts, p_sps->vui.i_time_scale,
340 p_sps->vui.i_num_units_in_tick );
343 if(!p_sys->b_invalid_pic_reorder_max && i_nal_type == H264_NAL_SLICE_IDR)
347 h264_get_dpb_values(p_sps, &i_reorder, &dummy);
348 vlc_mutex_lock(&p_sys->lock);
349 p_sys->i_pic_reorder_max = i_reorder;
350 pic_holder_update_reorder_max(p_sys->pic_holder,
351 p_sys->i_pic_reorder_max,
353 vlc_mutex_unlock(&p_sys->lock);
358 return true; /* No need to parse further NAL */
360 else if(i_nal_type == H264_NAL_SEI)
362 if(i_sei_count < VT_MAX_SEI_COUNT)
364 sei_array[i_sei_count].p_nal = p_nal;
365 sei_array[i_sei_count++].i_nal = i_nal;
374 static block_t *ProcessBlockH264(decoder_t *p_dec, block_t *p_block, bool *pb_config_changed)
376 decoder_sys_t *p_sys = p_dec->p_sys;
377 return p_sys->hh.pf_process_block(&p_sys->hh, p_block, pb_config_changed);
381 static bool InitH264(decoder_t *p_dec)
383 decoder_sys_t *p_sys = p_dec->p_sys;
384 h264_poc_context_init(&p_sys->h264_pocctx);
385 hxxx_helper_init(&p_sys->hh, VLC_OBJECT(p_dec),
386 p_dec->fmt_in.i_codec, true);
387 return hxxx_helper_set_extra(&p_sys->hh, p_dec->fmt_in.p_extra,
388 p_dec->fmt_in.i_extra) == VLC_SUCCESS;
391 static void CleanH264(decoder_t *p_dec)
393 decoder_sys_t *p_sys = p_dec->p_sys;
394 hxxx_helper_clean(&p_sys->hh);
397 static CFMutableDictionaryRef GetDecoderExtradataH264(decoder_t *p_dec)
399 decoder_sys_t *p_sys = p_dec->p_sys;
401 CFMutableDictionaryRef extradata = nil;
402 if (p_dec->fmt_in.i_extra && p_sys->hh.b_is_xvcC)
404 /* copy DecoderConfiguration */
405 extradata = ExtradataInfoCreate(CFSTR("avcC"),
406 p_dec->fmt_in.p_extra,
407 p_dec->fmt_in.i_extra);
409 else if (p_sys->hh.h264.i_pps_count && p_sys->hh.h264.i_sps_count)
411 /* build DecoderConfiguration from gathered */
412 block_t *p_avcC = h264_helper_get_avcc_config(&p_sys->hh);
415 extradata = ExtradataInfoCreate(CFSTR("avcC"),
418 block_Release(p_avcC);
424 static bool CodecSupportedH264(decoder_t *p_dec)
426 decoder_sys_t *p_sys = p_dec->p_sys;
428 uint8_t i_profile, i_level;
429 if (hxxx_helper_get_current_profile_level(&p_sys->hh, &i_profile, &i_level))
433 case PROFILE_H264_BASELINE:
434 case PROFILE_H264_MAIN:
435 case PROFILE_H264_HIGH:
438 case PROFILE_H264_HIGH_10:
440 if (deviceSupportsAdvancedProfiles())
444 msg_Err(p_dec, "current device doesn't support H264 10bits");
451 msg_Warn(p_dec, "unknown H264 profile %" PRIx8, i_profile);
456 /* A level higher than 5.2 was not tested, so don't dare to try to decode
457 * it. On SoC A8, 4.2 is the highest specified profile. on Twister, we can
459 if (i_level > 52 || (i_level > 42 && !deviceSupportsAdvancedLevels()))
461 msg_Err(p_dec, "current device doesn't support this H264 level: %"
466 HXXXGetBestChroma(p_dec);
471 static bool LateStartH264(decoder_t *p_dec)
473 decoder_sys_t *p_sys = p_dec->p_sys;
474 return (p_dec->fmt_in.i_extra == 0 &&
475 (!p_sys->hh.h264.i_pps_count || !p_sys->hh.h264.i_sps_count) );
478 static bool ConfigureVoutH264(decoder_t *p_dec)
480 decoder_sys_t *p_sys = p_dec->p_sys;
482 if(p_dec->fmt_in.video.primaries == COLOR_PRIMARIES_UNDEF)
484 video_color_primaries_t primaries;
485 video_transfer_func_t transfer;
486 video_color_space_t colorspace;
488 if (hxxx_helper_get_colorimetry(&p_sys->hh,
492 &full_range) == VLC_SUCCESS)
494 p_dec->fmt_out.video.primaries = primaries;
495 p_dec->fmt_out.video.transfer = transfer;
496 p_dec->fmt_out.video.space = colorspace;
497 p_dec->fmt_out.video.b_color_range_full = full_range;
501 if (!p_dec->fmt_in.video.i_visible_width || !p_dec->fmt_in.video.i_visible_height)
503 unsigned i_width, i_height, i_vis_width, i_vis_height;
505 hxxx_helper_get_current_picture_size(&p_sys->hh,
507 &i_vis_width, &i_vis_height))
509 p_dec->fmt_out.video.i_visible_width = i_vis_width;
510 p_dec->fmt_out.video.i_width = ALIGN_16( i_vis_width );
511 p_dec->fmt_out.video.i_visible_height = i_vis_height;
512 p_dec->fmt_out.video.i_height = ALIGN_16( i_vis_height );
517 if(!p_dec->fmt_in.video.i_sar_num || !p_dec->fmt_in.video.i_sar_den)
519 int i_sar_num, i_sar_den;
521 hxxx_helper_get_current_sar(&p_sys->hh, &i_sar_num, &i_sar_den))
523 p_dec->fmt_out.video.i_sar_num = i_sar_num;
524 p_dec->fmt_out.video.i_sar_den = i_sar_den;
531 static bool VideoToolboxNeedsToRestartH264(decoder_t *p_dec,
532 VTDecompressionSessionRef session)
534 decoder_sys_t *p_sys = p_dec->p_sys;
535 const struct hxxx_helper *hh = &p_sys->hh;
537 unsigned w, h, vw, vh;
540 if (hxxx_helper_get_current_picture_size(hh, &w, &h, &vw, &vh) != VLC_SUCCESS)
543 if (hxxx_helper_get_current_sar(hh, &sarn, &sard) != VLC_SUCCESS)
548 CFMutableDictionaryRef decoderConfiguration =
549 CreateSessionDescriptionFormat(p_dec, sarn, sard);
550 if (decoderConfiguration != nil)
552 CMFormatDescriptionRef newvideoFormatDesc;
553 /* create new video format description */
554 OSStatus status = CMVideoFormatDescriptionCreate(kCFAllocatorDefault,
557 decoderConfiguration,
558 &newvideoFormatDesc);
561 b_ret = !VTDecompressionSessionCanAcceptFormatDescription(session,
563 CFRelease(newvideoFormatDesc);
565 CFRelease(decoderConfiguration);
571 static bool InitHEVC(decoder_t *p_dec)
573 decoder_sys_t *p_sys = p_dec->p_sys;
574 hevc_poc_cxt_init(&p_sys->hevc_pocctx);
575 hxxx_helper_init(&p_sys->hh, VLC_OBJECT(p_dec),
576 p_dec->fmt_in.i_codec, true);
577 return hxxx_helper_set_extra(&p_sys->hh, p_dec->fmt_in.p_extra,
578 p_dec->fmt_in.i_extra) == VLC_SUCCESS;
581 #define CleanHEVC CleanH264
583 static void GetxPSHEVC(uint8_t i_id, void *priv,
584 hevc_picture_parameter_set_t **pp_pps,
585 hevc_sequence_parameter_set_t **pp_sps,
586 hevc_video_parameter_set_t **pp_vps)
588 decoder_sys_t *p_sys = priv;
590 *pp_pps = p_sys->hh.hevc.pps_list[i_id].hevc_pps;
598 uint8_t i_sps_id = hevc_get_pps_sps_id(*pp_pps);
599 *pp_sps = p_sys->hh.hevc.sps_list[i_sps_id].hevc_sps;
604 uint8_t i_vps_id = hevc_get_sps_vps_id(*pp_sps);
605 *pp_vps = p_sys->hh.hevc.vps_list[i_vps_id].hevc_vps;
610 struct hevc_sei_callback_s
612 hevc_sei_pic_timing_t *p_timing;
613 const hevc_sequence_parameter_set_t *p_sps;
616 static bool ParseHEVCSEI(const hxxx_sei_data_t *p_sei_data, void *priv)
618 if(p_sei_data->i_type == HXXX_SEI_PIC_TIMING)
620 struct hevc_sei_callback_s *s = priv;
622 s->p_timing = hevc_decode_sei_pic_timing(p_sei_data->p_bs, s->p_sps);
628 static bool FillReorderInfoHEVC(decoder_t *p_dec, const block_t *p_block,
629 frame_info_t *p_info)
631 decoder_sys_t *p_sys = p_dec->p_sys;
632 hxxx_iterator_ctx_t itctx;
633 hxxx_iterator_init(&itctx, p_block->p_buffer, p_block->i_buffer,
634 p_sys->hh.i_nal_length_size);
636 const uint8_t *p_nal; size_t i_nal;
639 const uint8_t *p_nal;
641 } sei_array[VT_MAX_SEI_COUNT];
642 size_t i_sei_count = 0;
644 while(hxxx_iterate_next(&itctx, &p_nal, &i_nal))
646 if(i_nal < 2 || hevc_getNALLayer(p_nal) > 0)
649 const enum hevc_nal_unit_type_e i_nal_type = hevc_getNALType(p_nal);
650 if (i_nal_type <= HEVC_NAL_IRAP_VCL23)
652 hevc_slice_segment_header_t *p_sli =
653 hevc_decode_slice_header(p_nal, i_nal, true, GetxPSHEVC, p_sys);
657 /* XXX: Work-around a VT bug on recent devices (iPhone X, MacBook
658 * Pro 2017). The VT session will report a BadDataErr if you send a
659 * RASL frame just after a CRA one. Indeed, RASL frames are
660 * corrupted if the decoding start at an IRAP frame (IDR/CRA), VT
661 * is likely failing to handle this case. */
662 if (!p_sys->b_vt_feed && (i_nal_type != HEVC_NAL_IDR_W_RADL &&
663 i_nal_type != HEVC_NAL_IDR_N_LP))
664 p_sys->b_drop_blocks = true;
665 else if (p_sys->b_drop_blocks)
667 if (i_nal_type == HEVC_NAL_RASL_N || i_nal_type == HEVC_NAL_RASL_R)
669 hevc_rbsp_release_slice_header(p_sli);
673 p_sys->b_drop_blocks = false;
676 p_info->b_keyframe = i_nal_type >= HEVC_NAL_BLA_W_LP;
678 hevc_sequence_parameter_set_t *p_sps;
679 hevc_picture_parameter_set_t *p_pps;
680 hevc_video_parameter_set_t *p_vps;
681 GetxPSHEVC(hevc_get_slice_pps_id(p_sli), p_sys, &p_pps, &p_sps, &p_vps);
684 struct hevc_sei_callback_s sei;
688 const int POC = hevc_compute_picture_order_count(p_sps, p_sli,
689 &p_sys->hevc_pocctx);
691 for(size_t i=0; i<i_sei_count; i++)
692 HxxxParseSEI(sei_array[i].p_nal, sei_array[i].i_nal,
693 2, ParseHEVCSEI, &sei);
696 p_info->i_foc = POC; /* clearly looks wrong :/ */
697 p_info->i_num_ts = hevc_get_num_clock_ts(p_sps, sei.p_timing);
698 p_info->b_flush = (POC == 0);
699 p_info->b_field = (p_info->i_num_ts == 1);
700 p_info->b_progressive = hevc_frame_is_progressive(p_sps, sei.p_timing);
702 /* Set frame rate for timings in case of missing rate */
703 if( (!p_dec->fmt_in.video.i_frame_rate_base ||
704 !p_dec->fmt_in.video.i_frame_rate) )
707 if(hevc_get_frame_rate(p_sps, p_vps, &num, &den))
708 date_Change(&p_sys->pts, num, den);
712 hevc_release_sei_pic_timing(sei.p_timing);
714 if(!p_sys->b_invalid_pic_reorder_max && p_vps)
716 vlc_mutex_lock(&p_sys->lock);
717 p_sys->i_pic_reorder_max = hevc_get_max_num_reorder(p_vps);
718 pic_holder_update_reorder_max(p_sys->pic_holder,
719 p_sys->i_pic_reorder_max,
721 vlc_mutex_unlock(&p_sys->lock);
726 hevc_rbsp_release_slice_header(p_sli);
727 return true; /* No need to parse further NAL */
729 else if(i_nal_type == HEVC_NAL_PREF_SEI)
731 if(i_sei_count < VT_MAX_SEI_COUNT)
733 sei_array[i_sei_count].p_nal = p_nal;
734 sei_array[i_sei_count++].i_nal = i_nal;
742 static CFMutableDictionaryRef GetDecoderExtradataHEVC(decoder_t *p_dec)
744 decoder_sys_t *p_sys = p_dec->p_sys;
746 CFMutableDictionaryRef extradata = nil;
747 if (p_dec->fmt_in.i_extra && p_sys->hh.b_is_xvcC)
749 /* copy DecoderConfiguration */
750 extradata = ExtradataInfoCreate(CFSTR("hvcC"),
751 p_dec->fmt_in.p_extra,
752 p_dec->fmt_in.i_extra);
754 else if (p_sys->hh.hevc.i_pps_count &&
755 p_sys->hh.hevc.i_sps_count &&
756 p_sys->hh.hevc.i_vps_count)
758 /* build DecoderConfiguration from gathered */
759 block_t *p_hvcC = hevc_helper_get_hvcc_config(&p_sys->hh);
762 extradata = ExtradataInfoCreate(CFSTR("hvcC"),
765 block_Release(p_hvcC);
771 static bool LateStartHEVC(decoder_t *p_dec)
773 decoder_sys_t *p_sys = p_dec->p_sys;
774 return (p_dec->fmt_in.i_extra == 0 &&
775 (!p_sys->hh.hevc.i_pps_count ||
776 !p_sys->hh.hevc.i_sps_count ||
777 !p_sys->hh.hevc.i_vps_count) );
780 static bool CodecSupportedHEVC(decoder_t *p_dec)
782 HXXXGetBestChroma(p_dec);
787 #define ConfigureVoutHEVC ConfigureVoutH264
788 #define ProcessBlockHEVC ProcessBlockH264
789 #define VideoToolboxNeedsToRestartHEVC VideoToolboxNeedsToRestartH264
791 static CFMutableDictionaryRef GetDecoderExtradataMPEG4(decoder_t *p_dec)
793 if (p_dec->fmt_in.i_extra)
794 return ESDSExtradataInfoCreate(p_dec, p_dec->fmt_in.p_extra,
795 p_dec->fmt_in.i_extra);
797 return nil; /* MPEG4 without esds ? */
800 static CFMutableDictionaryRef GetDecoderExtradataDefault(decoder_t *p_dec)
802 return ExtradataInfoCreate(NULL, NULL, 0); /* Empty Needed ? */
805 /* !Codec Specific */
807 static void InsertIntoDPB(decoder_sys_t *p_sys, frame_info_t *p_info)
809 frame_info_t **pp_lead_in = &p_sys->p_pic_reorder;
811 for( ;; pp_lead_in = & ((*pp_lead_in)->p_next))
814 if(*pp_lead_in == NULL)
816 else if(p_sys->b_poc_based_reorder)
817 b_insert = ((*pp_lead_in)->i_foc > p_info->i_foc);
819 b_insert = ((*pp_lead_in)->p_picture->date >= p_info->p_picture->date);
823 p_info->p_next = *pp_lead_in;
824 *pp_lead_in = p_info;
825 p_sys->i_pic_reorder += (p_info->b_field) ? 1 : 2;
830 for(frame_info_t *p_in=p_sys->p_pic_reorder; p_in; p_in = p_in->p_next)
831 printf(" %d", p_in->i_foc);
836 static picture_t * RemoveOneFrameFromDPB(decoder_sys_t *p_sys)
838 frame_info_t *p_info = p_sys->p_pic_reorder;
842 const int i_framepoc = p_info->i_poc;
844 picture_t *p_ret = NULL;
845 picture_t **pp_ret_last = &p_ret;
850 picture_t *p_field = p_info->p_picture;
852 /* Compute time if missing */
853 if (p_field->date == VLC_TICK_INVALID)
854 p_field->date = date_Get(&p_sys->pts);
856 date_Set(&p_sys->pts, p_field->date);
858 /* Set next picture time, in case it is missing */
859 if (p_info->i_length)
860 date_Set(&p_sys->pts, p_field->date + p_info->i_length);
862 date_Increment(&p_sys->pts, p_info->i_num_ts);
864 *pp_ret_last = p_field;
865 pp_ret_last = &p_field->p_next;
867 p_sys->i_pic_reorder -= (p_info->b_field) ? 1 : 2;
869 p_sys->p_pic_reorder = p_info->p_next;
871 p_info = p_sys->p_pic_reorder;
875 if (p_sys->b_poc_based_reorder)
876 b_dequeue = (p_info->i_poc == i_framepoc);
878 b_dequeue = (p_field->date == p_info->p_picture->date);
880 else b_dequeue = false;
887 static void DrainDPBLocked(decoder_t *p_dec, bool flush)
889 decoder_sys_t *p_sys = p_dec->p_sys;
892 picture_t *p_fields = RemoveOneFrameFromDPB(p_sys);
893 if (p_fields == NULL)
897 picture_t *p_next = p_fields->p_next;
898 p_fields->p_next = NULL;
900 picture_Release(p_fields);
902 decoder_QueueVideo(p_dec, p_fields);
904 } while(p_fields != NULL);
908 static frame_info_t * CreateReorderInfo(decoder_t *p_dec, const block_t *p_block)
910 decoder_sys_t *p_sys = p_dec->p_sys;
911 frame_info_t *p_info = calloc(1, sizeof(*p_info));
915 if (p_sys->pf_fill_reorder_info)
917 if(!p_sys->pf_fill_reorder_info(p_dec, p_block, p_info))
925 p_info->i_num_ts = 2;
926 p_info->b_progressive = true;
927 p_info->b_field = false;
928 p_info->b_keyframe = true;
931 p_info->i_length = p_block->i_length;
933 /* required for still pictures/menus */
934 p_info->b_forced = (p_block->i_flags & BLOCK_FLAG_END_OF_SEQUENCE);
936 if (date_Get(&p_sys->pts) == VLC_TICK_INVALID)
937 date_Set(&p_sys->pts, p_block->i_dts);
942 static void OnDecodedFrame(decoder_t *p_dec, frame_info_t *p_info)
944 decoder_sys_t *p_sys = p_dec->p_sys;
945 assert(p_info->p_picture);
946 while(p_info->b_flush || p_sys->i_pic_reorder >= (p_sys->i_pic_reorder_max * 2))
948 /* First check if DPB sizing was correct before removing one frame */
949 if (p_sys->p_pic_reorder && !p_info->b_flush &&
950 p_sys->i_pic_reorder_max < H264_MAX_DPB)
952 if(p_sys->b_poc_based_reorder && p_sys->p_pic_reorder->i_foc > p_info->i_foc)
954 p_sys->b_invalid_pic_reorder_max = true;
955 p_sys->i_pic_reorder_max++;
956 pic_holder_update_reorder_max(p_sys->pic_holder,
957 p_sys->i_pic_reorder_max, p_info->i_num_ts);
958 msg_Info(p_dec, "Raising max DPB to %"PRIu8, p_sys->i_pic_reorder_max);
961 else if (!p_sys->b_poc_based_reorder &&
962 p_info->p_picture->date > VLC_TICK_INVALID &&
963 p_sys->p_pic_reorder->p_picture->date > p_info->p_picture->date)
965 p_sys->b_invalid_pic_reorder_max = true;
966 p_sys->i_pic_reorder_max++;
967 pic_holder_update_reorder_max(p_sys->pic_holder,
968 p_sys->i_pic_reorder_max, p_info->i_num_ts);
969 msg_Info(p_dec, "Raising max DPB to %"PRIu8, p_sys->i_pic_reorder_max);
974 picture_t *p_fields = RemoveOneFrameFromDPB(p_sys);
975 if (p_fields == NULL)
979 picture_t *p_next = p_fields->p_next;
980 p_fields->p_next = NULL;
981 decoder_QueueVideo(p_dec, p_fields);
983 } while(p_fields != NULL);
986 InsertIntoDPB(p_sys, p_info);
989 static CMVideoCodecType CodecPrecheck(decoder_t *p_dec)
991 decoder_sys_t *p_sys = p_dec->p_sys;
993 /* check for the codec we can and want to decode */
994 switch (p_dec->fmt_in.i_codec) {
996 return kCMVideoCodecType_H264;
999 if (!deviceSupportsHEVC())
1001 msg_Warn(p_dec, "device doesn't support HEVC");
1004 return kCMVideoCodecType_HEVC;
1006 case VLC_CODEC_MP4V:
1008 if (p_dec->fmt_in.i_original_fourcc == VLC_FOURCC( 'X','V','I','D' )) {
1009 msg_Warn(p_dec, "XVID decoding not implemented, fallback on software");
1013 msg_Dbg(p_dec, "Will decode MP4V with original FourCC '%4.4s'", (char *)&p_dec->fmt_in.i_original_fourcc);
1014 return kCMVideoCodecType_MPEG4Video;
1016 #if !TARGET_OS_IPHONE
1017 case VLC_CODEC_H263:
1018 return kCMVideoCodecType_H263;
1020 /* there are no DV or ProRes decoders on iOS, so bailout early */
1021 case VLC_CODEC_PRORES:
1022 /* the VT decoder can't differenciate between the ProRes flavors, so we do it */
1023 switch (p_dec->fmt_in.i_original_fourcc) {
1024 case VLC_FOURCC( 'a','p','4','c' ):
1025 case VLC_FOURCC( 'a','p','4','h' ):
1026 case VLC_FOURCC( 'a','p','4','x' ):
1027 return kCMVideoCodecType_AppleProRes4444;
1029 case VLC_FOURCC( 'a','p','c','h' ):
1030 return kCMVideoCodecType_AppleProRes422HQ;
1032 case VLC_FOURCC( 'a','p','c','s' ):
1033 return kCMVideoCodecType_AppleProRes422LT;
1035 case VLC_FOURCC( 'a','p','c','o' ):
1036 return kCMVideoCodecType_AppleProRes422Proxy;
1039 return kCMVideoCodecType_AppleProRes422;
1043 /* the VT decoder can't differenciate between PAL and NTSC, so we need to do it */
1044 switch (p_dec->fmt_in.i_original_fourcc) {
1045 case VLC_FOURCC( 'd', 'v', 'c', ' '):
1046 case VLC_FOURCC( 'd', 'v', ' ', ' '):
1047 msg_Dbg(p_dec, "Decoding DV NTSC");
1048 return kCMVideoCodecType_DVCNTSC;
1050 case VLC_FOURCC( 'd', 'v', 's', 'd'):
1051 case VLC_FOURCC( 'd', 'v', 'c', 'p'):
1052 case VLC_FOURCC( 'D', 'V', 'S', 'D'):
1053 msg_Dbg(p_dec, "Decoding DV PAL");
1054 return kCMVideoCodecType_DVCPAL;
1059 /* mpgv / mp2v needs fixing, so disable it for now */
1061 case VLC_CODEC_MPGV:
1062 return kCMVideoCodecType_MPEG1Video;
1063 case VLC_CODEC_MP2V:
1064 return kCMVideoCodecType_MPEG2Video;
1069 msg_Err(p_dec, "'%4.4s' is not supported", (char *)&p_dec->fmt_in.i_codec);
1074 vlc_assert_unreachable();
1077 static CFMutableDictionaryRef CreateSessionDescriptionFormat(decoder_t *p_dec,
1081 decoder_sys_t *p_sys = p_dec->p_sys;
1083 CFMutableDictionaryRef decoderConfiguration = cfdict_create(0);
1084 if (decoderConfiguration == NULL)
1087 CFMutableDictionaryRef extradata = p_sys->pf_get_extradata
1088 ? p_sys->pf_get_extradata(p_dec) : nil;
1091 /* then decoder will also fail if required, no need to handle it */
1092 CFDictionarySetValue(decoderConfiguration,
1093 kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms,
1095 CFRelease(extradata);
1098 CFDictionarySetValue(decoderConfiguration,
1099 kCVImageBufferChromaLocationBottomFieldKey,
1100 kCVImageBufferChromaLocation_Left);
1101 CFDictionarySetValue(decoderConfiguration,
1102 kCVImageBufferChromaLocationTopFieldKey,
1103 kCVImageBufferChromaLocation_Left);
1105 /* pixel aspect ratio */
1106 if(i_sar_num && i_sar_den)
1108 CFMutableDictionaryRef pixelaspectratio = cfdict_create(2);
1109 if(pixelaspectratio == NULL)
1111 CFRelease(decoderConfiguration);
1115 cfdict_set_int32(pixelaspectratio,
1116 kCVImageBufferPixelAspectRatioHorizontalSpacingKey,
1118 cfdict_set_int32(pixelaspectratio,
1119 kCVImageBufferPixelAspectRatioVerticalSpacingKey,
1121 CFDictionarySetValue(decoderConfiguration,
1122 kCVImageBufferPixelAspectRatioKey,
1124 CFRelease(pixelaspectratio);
1127 /* Setup YUV->RGB matrix since VT can output BGRA directly. Don't setup
1128 * transfer and primaries since the transformation is done via the GL
1129 * fragment shader. */
1130 CFStringRef yuvmatrix;
1131 switch (p_dec->fmt_out.video.space)
1133 case COLOR_SPACE_BT601:
1134 yuvmatrix = kCVImageBufferYCbCrMatrix_ITU_R_601_4;
1136 case COLOR_SPACE_BT2020:
1137 #pragma clang diagnostic push
1138 #pragma clang diagnostic ignored "-Wpartial-availability"
1139 if (&kCVImageBufferColorPrimaries_ITU_R_2020 != nil)
1141 yuvmatrix = kCVImageBufferColorPrimaries_ITU_R_2020;
1144 #pragma clang diagnostic pop
1146 case COLOR_SPACE_BT709:
1148 yuvmatrix = kCVImageBufferColorPrimaries_ITU_R_709_2;
1151 CFDictionarySetValue(decoderConfiguration, kCVImageBufferYCbCrMatrixKey,
1154 #pragma clang diagnostic push
1155 #pragma clang diagnostic ignored "-Wpartial-availability"
1157 /* enable HW accelerated playback, since this is optional on OS X
1158 * note that the backend may still fallback on software mode if no
1159 * suitable hardware is available */
1160 CFDictionarySetValue(decoderConfiguration,
1161 kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder,
1164 /* on OS X, we can force VT to fail if no suitable HW decoder is available,
1165 * preventing the aforementioned SW fallback */
1166 if (var_InheritBool(p_dec, "videotoolbox-hw-decoder-only"))
1167 CFDictionarySetValue(decoderConfiguration,
1168 kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder,
1171 #pragma clang diagnostic pop
1173 CFDictionarySetValue(decoderConfiguration,
1174 kVTDecompressionPropertyKey_FieldMode,
1175 kVTDecompressionProperty_FieldMode_DeinterlaceFields);
1176 CFDictionarySetValue(decoderConfiguration,
1177 kVTDecompressionPropertyKey_DeinterlaceMode,
1178 kVTDecompressionProperty_DeinterlaceMode_Temporal);
1180 return decoderConfiguration;
1183 static void PtsInit(decoder_t *p_dec)
1185 decoder_sys_t *p_sys = p_dec->p_sys;
1187 if( p_dec->fmt_in.video.i_frame_rate_base && p_dec->fmt_in.video.i_frame_rate )
1189 date_Init( &p_sys->pts, p_dec->fmt_in.video.i_frame_rate * 2,
1190 p_dec->fmt_in.video.i_frame_rate_base );
1192 else date_Init( &p_sys->pts, 2 * 30000, 1001 );
1195 static int StartVideoToolbox(decoder_t *p_dec)
1197 decoder_sys_t *p_sys = p_dec->p_sys;
1200 if(p_sys->pf_late_start && p_sys->pf_late_start(p_dec))
1202 assert(p_sys->session == NULL);
1206 /* Fills fmt_out (from extradata if any) */
1207 if(ConfigureVout(p_dec) != VLC_SUCCESS)
1208 return VLC_EGENERIC;
1210 /* destination pixel buffer attributes */
1211 CFMutableDictionaryRef destinationPixelBufferAttributes = cfdict_create(0);
1212 if(destinationPixelBufferAttributes == nil)
1213 return VLC_EGENERIC;
1215 CFMutableDictionaryRef decoderConfiguration =
1216 CreateSessionDescriptionFormat(p_dec,
1217 p_dec->fmt_out.video.i_sar_num,
1218 p_dec->fmt_out.video.i_sar_den);
1219 if(decoderConfiguration == nil)
1221 CFRelease(destinationPixelBufferAttributes);
1222 return VLC_EGENERIC;
1225 /* create video format description */
1226 OSStatus status = CMVideoFormatDescriptionCreate(
1227 kCFAllocatorDefault,
1229 p_dec->fmt_out.video.i_visible_width,
1230 p_dec->fmt_out.video.i_visible_height,
1231 decoderConfiguration,
1232 &p_sys->videoFormatDescription);
1235 CFRelease(destinationPixelBufferAttributes);
1236 CFRelease(decoderConfiguration);
1237 msg_Err(p_dec, "video format description creation failed (%i)", (int)status);
1238 return VLC_EGENERIC;
1241 #if !TARGET_OS_IPHONE
1242 CFDictionarySetValue(destinationPixelBufferAttributes,
1243 kCVPixelBufferIOSurfaceOpenGLTextureCompatibilityKey,
1246 CFDictionarySetValue(destinationPixelBufferAttributes,
1247 kCVPixelBufferOpenGLESCompatibilityKey,
1251 cfdict_set_int32(destinationPixelBufferAttributes,
1252 kCVPixelBufferWidthKey, p_dec->fmt_out.video.i_visible_width);
1253 cfdict_set_int32(destinationPixelBufferAttributes,
1254 kCVPixelBufferHeightKey, p_dec->fmt_out.video.i_visible_height);
1256 if (p_sys->i_cvpx_format != 0)
1258 int chroma = htonl(p_sys->i_cvpx_format);
1259 msg_Warn(p_dec, "forcing CVPX format: %4.4s", (const char *) &chroma);
1260 cfdict_set_int32(destinationPixelBufferAttributes,
1261 kCVPixelBufferPixelFormatTypeKey,
1262 p_sys->i_cvpx_format);
1265 cfdict_set_int32(destinationPixelBufferAttributes,
1266 kCVPixelBufferBytesPerRowAlignmentKey, 16);
1268 /* setup decoder callback record */
1269 VTDecompressionOutputCallbackRecord decoderCallbackRecord;
1270 decoderCallbackRecord.decompressionOutputCallback = DecoderCallback;
1271 decoderCallbackRecord.decompressionOutputRefCon = p_dec;
1273 /* create decompression session */
1274 status = VTDecompressionSessionCreate(kCFAllocatorDefault,
1275 p_sys->videoFormatDescription,
1276 decoderConfiguration,
1277 destinationPixelBufferAttributes,
1278 &decoderCallbackRecord, &p_sys->session);
1279 CFRelease(decoderConfiguration);
1280 CFRelease(destinationPixelBufferAttributes);
1282 if (HandleVTStatus(p_dec, status, NULL) != VLC_SUCCESS)
1283 return VLC_EGENERIC;
1290 static void StopVideoToolbox(decoder_t *p_dec)
1292 decoder_sys_t *p_sys = p_dec->p_sys;
1294 if (p_sys->session != nil)
1298 VTDecompressionSessionInvalidate(p_sys->session);
1299 CFRelease(p_sys->session);
1300 p_sys->session = nil;
1302 #if TARGET_OS_IPHONE
1303 /* In case of 4K 10bits (BGRA), we can easily reach the device max
1304 * memory when flushing. Indeed, we'll create a new VT session that
1305 * will reallocate frames while previous frames are still used by the
1306 * vout (and not released). To work-around this issue, we force a vout
1308 if (p_dec->fmt_out.i_codec == VLC_CODEC_CVPX_BGRA
1309 && p_dec->fmt_out.video.i_width * p_dec->fmt_out.video.i_height >= 8000000)
1311 const video_format_t orig = p_dec->fmt_out.video;
1312 p_dec->fmt_out.video.i_width = p_dec->fmt_out.video.i_height =
1313 p_dec->fmt_out.video.i_visible_width = p_dec->fmt_out.video.i_visible_height = 64;
1314 (void) decoder_UpdateVideoFormat(p_dec);
1315 p_dec->fmt_out.video = orig;
1319 p_sys->b_format_propagated = false;
1320 p_dec->fmt_out.i_codec = 0;
1323 if (p_sys->videoFormatDescription != nil) {
1324 CFRelease(p_sys->videoFormatDescription);
1325 p_sys->videoFormatDescription = nil;
1327 p_sys->b_vt_feed = false;
1328 p_sys->b_drop_blocks = false;
1331 #pragma mark - module open and close
1334 static int OpenDecoder(vlc_object_t *p_this)
1336 decoder_t *p_dec = (decoder_t *)p_this;
1338 if (!var_InheritBool(p_dec, "videotoolbox"))
1339 return VLC_EGENERIC;
1341 #if TARGET_OS_IPHONE
1342 if (unlikely([[UIDevice currentDevice].systemVersion floatValue] < 8.0)) {
1343 msg_Warn(p_dec, "decoder skipped as OS is too old");
1344 return VLC_EGENERIC;
1348 /* Fail if this module already failed to decode this ES */
1349 if (var_Type(p_dec, "videotoolbox-failed") != 0)
1350 return VLC_EGENERIC;
1352 /* check quickly if we can digest the offered data */
1353 CMVideoCodecType codec;
1355 codec = CodecPrecheck(p_dec);
1357 return VLC_EGENERIC;
1359 /* now that we see a chance to decode anything, allocate the
1360 * internals and start the decoding session */
1361 decoder_sys_t *p_sys;
1362 p_sys = calloc(1, sizeof(*p_sys));
1365 p_dec->p_sys = p_sys;
1366 p_sys->session = nil;
1367 p_sys->codec = codec;
1368 p_sys->videoFormatDescription = nil;
1369 p_sys->i_pic_reorder_max = 4;
1370 p_sys->vtsession_status = VTSESSION_STATUS_OK;
1371 p_sys->b_cvpx_format_forced = false;
1373 char *cvpx_chroma = var_InheritString(p_dec, "videotoolbox-cvpx-chroma");
1374 if (cvpx_chroma != NULL)
1376 if (strlen(cvpx_chroma) != 4)
1378 msg_Err(p_dec, "invalid videotoolbox-cvpx-chroma option");
1381 return VLC_EGENERIC;
1383 memcpy(&p_sys->i_cvpx_format, cvpx_chroma, 4);
1384 p_sys->i_cvpx_format = ntohl(p_sys->i_cvpx_format);
1385 p_sys->b_cvpx_format_forced = true;
1389 p_sys->pic_holder = malloc(sizeof(struct pic_holder));
1390 if (!p_sys->pic_holder)
1396 vlc_mutex_init(&p_sys->pic_holder->lock);
1397 vlc_cond_init(&p_sys->pic_holder->wait);
1398 p_sys->pic_holder->nb_field_out = 0;
1399 p_sys->pic_holder->closed = false;
1400 p_sys->pic_holder->field_reorder_max = p_sys->i_pic_reorder_max * 2;
1401 p_sys->b_vt_need_keyframe = false;
1403 vlc_mutex_init(&p_sys->lock);
1405 p_dec->pf_decode = DecodeBlock;
1406 p_dec->pf_flush = RequestFlush;
1410 case kCMVideoCodecType_H264:
1411 p_sys->pf_codec_init = InitH264;
1412 p_sys->pf_codec_clean = CleanH264;
1413 p_sys->pf_codec_supported = CodecSupportedH264;
1414 p_sys->pf_late_start = LateStartH264;
1415 p_sys->pf_process_block = ProcessBlockH264;
1416 p_sys->pf_need_restart = VideoToolboxNeedsToRestartH264;
1417 p_sys->pf_configure_vout = ConfigureVoutH264;
1418 p_sys->pf_get_extradata = GetDecoderExtradataH264;
1419 p_sys->pf_fill_reorder_info = FillReorderInfoH264;
1420 p_sys->b_poc_based_reorder = true;
1423 case kCMVideoCodecType_HEVC:
1424 p_sys->pf_codec_init = InitHEVC;
1425 p_sys->pf_codec_clean = CleanHEVC;
1426 p_sys->pf_codec_supported = CodecSupportedHEVC;
1427 p_sys->pf_late_start = LateStartHEVC;
1428 p_sys->pf_process_block = ProcessBlockHEVC;
1429 p_sys->pf_need_restart = VideoToolboxNeedsToRestartHEVC;
1430 p_sys->pf_configure_vout = ConfigureVoutHEVC;
1431 p_sys->pf_get_extradata = GetDecoderExtradataHEVC;
1432 p_sys->pf_fill_reorder_info = FillReorderInfoHEVC;
1433 p_sys->b_poc_based_reorder = true;
1434 p_sys->b_vt_need_keyframe = true;
1437 case kCMVideoCodecType_MPEG4Video:
1438 p_sys->pf_get_extradata = GetDecoderExtradataMPEG4;
1442 p_sys->pf_get_extradata = GetDecoderExtradataDefault;
1446 if (p_sys->pf_codec_init && !p_sys->pf_codec_init(p_dec))
1448 CloseDecoder(p_this);
1449 return VLC_EGENERIC;
1451 if (p_sys->pf_codec_supported && !p_sys->pf_codec_supported(p_dec))
1453 CloseDecoder(p_this);
1454 return VLC_EGENERIC;
1457 int i_ret = StartVideoToolbox(p_dec);
1458 if (i_ret == VLC_SUCCESS)
1459 msg_Info(p_dec, "Using Video Toolbox to decode '%4.4s'",
1460 (char *)&p_dec->fmt_in.i_codec);
1462 CloseDecoder(p_this);
1466 static void pic_holder_clean(struct pic_holder *pic_holder)
1468 vlc_mutex_destroy(&pic_holder->lock);
1469 vlc_cond_destroy(&pic_holder->wait);
1473 static void CloseDecoder(vlc_object_t *p_this)
1475 decoder_t *p_dec = (decoder_t *)p_this;
1476 decoder_sys_t *p_sys = p_dec->p_sys;
1478 StopVideoToolbox(p_dec);
1480 if(p_sys->pf_codec_clean)
1481 p_sys->pf_codec_clean(p_dec);
1483 vlc_mutex_destroy(&p_sys->lock);
1485 vlc_mutex_lock(&p_sys->pic_holder->lock);
1486 if (p_sys->pic_holder->nb_field_out == 0)
1488 vlc_mutex_unlock(&p_sys->pic_holder->lock);
1489 pic_holder_clean(p_sys->pic_holder);
1493 p_sys->pic_holder->closed = true;
1494 vlc_mutex_unlock(&p_sys->pic_holder->lock);
1499 #pragma mark - helpers
1501 static BOOL deviceSupportsHEVC()
1503 #pragma clang diagnostic push
1504 #pragma clang diagnostic ignored "-Wpartial-availability"
1506 #if (TARGET_OS_OSX && MAC_OS_X_VERSION_MAX_ALLOWED >= 101300) || \
1507 (TARGET_OS_IPHONE && __IPHONE_OS_VERSION_MAX_ALLOWED >= 110000) || \
1508 (TARGET_OS_TV && __TV_OS_VERSION_MAX_ALLOWED >= 110000)
1509 if (VTIsHardwareDecodeSupported != nil)
1510 return VTIsHardwareDecodeSupported(kCMVideoCodecType_HEVC);
1513 #pragma clang diagnostic pop
1517 static BOOL deviceSupportsAdvancedProfiles()
1519 #if TARGET_OS_IPHONE
1523 size = sizeof(type);
1524 sysctlbyname("hw.cputype", &type, &size, NULL, 0);
1526 /* Support for H264 profile HIGH 10 was introduced with the first 64bit Apple ARM SoC, the A7 */
1527 if (type == CPU_TYPE_ARM64)
1534 static BOOL deviceSupportsAdvancedLevels()
1536 #if TARGET_OS_IPHONE
1540 size = sizeof(cpufamily);
1541 sysctlbyname("hw.cpufamily", &cpufamily, &size, NULL, 0);
1543 /* Proper 4K decoding requires a Twister SoC
1544 * Everything below will kill the decoder daemon */
1545 if (cpufamily == CPUFAMILY_ARM_CYCLONE || cpufamily == CPUFAMILY_ARM_TYPHOON) {
1551 /* we need a 64bit SoC for advanced levels */
1559 static inline void bo_add_mp4_tag_descr(bo_t *p_bo, uint8_t tag, uint32_t size)
1561 bo_add_8(p_bo, tag);
1562 for (int i = 3; i>0; i--)
1563 bo_add_8(p_bo, (size>>(7*i)) | 0x80);
1564 bo_add_8(p_bo, size & 0x7F);
1567 static CFMutableDictionaryRef ESDSExtradataInfoCreate(decoder_t *p_dec,
1569 uint32_t i_buf_size)
1571 decoder_sys_t *p_sys = p_dec->p_sys;
1573 int full_size = 3 + 5 +13 + 5 + i_buf_size + 3;
1574 int config_size = 13 + 5 + i_buf_size;
1578 bool status = bo_init(&bo, 1024);
1582 bo_add_8(&bo, 0); // Version
1583 bo_add_24be(&bo, 0); // Flags
1585 // elementary stream description tag
1586 bo_add_mp4_tag_descr(&bo, 0x03, full_size);
1587 bo_add_16be(&bo, 0); // esid
1588 bo_add_8(&bo, 0); // stream priority (0-3)
1590 // decoder configuration description tag
1591 bo_add_mp4_tag_descr(&bo, 0x04, config_size);
1592 bo_add_8(&bo, 32); // object type identification (32 == MPEG4)
1593 bo_add_8(&bo, 0x11); // stream type
1594 bo_add_24be(&bo, 0); // buffer size
1595 bo_add_32be(&bo, 0); // max bitrate
1596 bo_add_32be(&bo, 0); // avg bitrate
1598 // decoder specific description tag
1599 bo_add_mp4_tag_descr(&bo, 0x05, i_buf_size);
1600 bo_add_mem(&bo, i_buf_size, p_buf);
1602 // sync layer configuration description tag
1603 bo_add_8(&bo, 0x06); // tag
1604 bo_add_8(&bo, 0x01); // length
1605 bo_add_8(&bo, 0x02); // no SL
1607 CFMutableDictionaryRef extradataInfo =
1608 ExtradataInfoCreate(CFSTR("esds"), bo.b->p_buffer, bo.b->i_buffer);
1610 return extradataInfo;
1613 static int ConfigureVout(decoder_t *p_dec)
1615 decoder_sys_t *p_sys = p_dec->p_sys;
1617 /* return our proper VLC internal state */
1618 p_dec->fmt_out.video = p_dec->fmt_in.video;
1619 p_dec->fmt_out.video.p_palette = NULL;
1620 p_dec->fmt_out.i_codec = 0;
1622 if(p_sys->pf_configure_vout &&
1623 !p_sys->pf_configure_vout(p_dec))
1624 return VLC_EGENERIC;
1626 if (!p_dec->fmt_out.video.i_sar_num || !p_dec->fmt_out.video.i_sar_den)
1628 p_dec->fmt_out.video.i_sar_num = 1;
1629 p_dec->fmt_out.video.i_sar_den = 1;
1632 if (!p_dec->fmt_out.video.i_visible_width || !p_dec->fmt_out.video.i_visible_height)
1634 p_dec->fmt_out.video.i_visible_width = p_dec->fmt_out.video.i_width;
1635 p_dec->fmt_out.video.i_visible_height = p_dec->fmt_out.video.i_height;
1638 p_dec->fmt_out.video.i_width = ALIGN_16( p_dec->fmt_out.video.i_visible_width );
1639 p_dec->fmt_out.video.i_height = ALIGN_16( p_dec->fmt_out.video.i_visible_height );
1644 static CFMutableDictionaryRef ExtradataInfoCreate(CFStringRef name,
1645 void *p_data, size_t i_data)
1647 CFMutableDictionaryRef extradataInfo = cfdict_create(1);
1648 if (extradataInfo == nil)
1654 CFDataRef extradata = CFDataCreate(kCFAllocatorDefault, p_data, i_data);
1655 if (extradata == nil)
1657 CFRelease(extradataInfo);
1660 CFDictionarySetValue(extradataInfo, name, extradata);
1661 CFRelease(extradata);
1662 return extradataInfo;
1665 static CMSampleBufferRef VTSampleBufferCreate(decoder_t *p_dec,
1666 CMFormatDescriptionRef fmt_desc,
1669 decoder_sys_t *p_sys = p_dec->p_sys;
1671 CMBlockBufferRef block_buf = NULL;
1672 CMSampleBufferRef sample_buf = NULL;
1674 if(!p_sys->b_poc_based_reorder && p_block->i_pts == VLC_TICK_INVALID)
1675 pts = CMTimeMake(p_block->i_dts, CLOCK_FREQ);
1677 pts = CMTimeMake(p_block->i_pts, CLOCK_FREQ);
1679 CMSampleTimingInfo timeInfoArray[1] = { {
1680 .duration = CMTimeMake(p_block->i_length, 1),
1681 .presentationTimeStamp = pts,
1682 .decodeTimeStamp = CMTimeMake(p_block->i_dts, CLOCK_FREQ),
1685 status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault,// structureAllocator
1686 p_block->p_buffer, // memoryBlock
1687 p_block->i_buffer, // blockLength
1688 kCFAllocatorNull, // blockAllocator
1689 NULL, // customBlockSource
1691 p_block->i_buffer, // dataLength
1696 status = CMSampleBufferCreate(kCFAllocatorDefault, // allocator
1697 block_buf, // dataBuffer
1699 0, // makeDataReadyCallback
1700 0, // makeDataReadyRefcon
1701 fmt_desc, // formatDescription
1703 1, // numSampleTimingEntries
1704 timeInfoArray, // sampleTimingArray
1705 0, // numSampleSizeEntries
1706 NULL, // sampleSizeArray
1708 if (status != noErr)
1709 msg_Warn(p_dec, "sample buffer creation failure %i", (int)status);
1711 msg_Warn(p_dec, "cm block buffer creation failure %i", (int)status);
1713 if (block_buf != nil)
1714 CFRelease(block_buf);
1720 static int HandleVTStatus(decoder_t *p_dec, OSStatus status,
1721 enum vtsession_status * p_vtsession_status)
1723 decoder_sys_t *p_sys = p_dec->p_sys;
1725 #define VTERRCASE(x) \
1726 case x: msg_Warn(p_dec, "vt session error: '" #x "'"); break;
1733 VTERRCASE(kVTPropertyNotSupportedErr)
1734 VTERRCASE(kVTPropertyReadOnlyErr)
1735 VTERRCASE(kVTParameterErr)
1736 VTERRCASE(kVTInvalidSessionErr)
1737 VTERRCASE(kVTAllocationFailedErr)
1738 VTERRCASE(kVTPixelTransferNotSupportedErr)
1739 VTERRCASE(kVTCouldNotFindVideoDecoderErr)
1740 VTERRCASE(kVTCouldNotCreateInstanceErr)
1741 VTERRCASE(kVTCouldNotFindVideoEncoderErr)
1742 VTERRCASE(kVTVideoDecoderBadDataErr)
1743 VTERRCASE(kVTVideoDecoderUnsupportedDataFormatErr)
1744 VTERRCASE(kVTVideoDecoderMalfunctionErr)
1745 VTERRCASE(kVTVideoEncoderMalfunctionErr)
1746 VTERRCASE(kVTVideoDecoderNotAvailableNowErr)
1747 VTERRCASE(kVTImageRotationNotSupportedErr)
1748 VTERRCASE(kVTVideoEncoderNotAvailableNowErr)
1749 VTERRCASE(kVTFormatDescriptionChangeNotSupportedErr)
1750 VTERRCASE(kVTInsufficientSourceColorDataErr)
1751 VTERRCASE(kVTCouldNotCreateColorCorrectionDataErr)
1752 VTERRCASE(kVTColorSyncTransformConvertFailedErr)
1753 VTERRCASE(kVTVideoDecoderAuthorizationErr)
1754 VTERRCASE(kVTVideoEncoderAuthorizationErr)
1755 VTERRCASE(kVTColorCorrectionPixelTransferFailedErr)
1756 VTERRCASE(kVTMultiPassStorageIdentifierMismatchErr)
1757 VTERRCASE(kVTMultiPassStorageInvalidErr)
1758 VTERRCASE(kVTFrameSiloInvalidTimeStampErr)
1759 VTERRCASE(kVTFrameSiloInvalidTimeRangeErr)
1760 VTERRCASE(kVTCouldNotFindTemporalFilterErr)
1761 VTERRCASE(kVTPixelTransferNotPermittedErr)
1763 msg_Warn(p_dec, "vt session error: "
1764 "'kVTColorCorrectionImageRotationFailedErr'");
1767 msg_Warn(p_dec, "unknown vt session error (%i)", (int)status);
1771 if (p_vtsession_status)
1775 case kVTPixelTransferNotSupportedErr:
1776 case kVTPixelTransferNotPermittedErr:
1777 *p_vtsession_status = VTSESSION_STATUS_RESTART_CHROMA;
1779 case -8960 /* codecErr */:
1780 case kVTVideoDecoderMalfunctionErr:
1781 case kVTInvalidSessionErr:
1782 *p_vtsession_status = VTSESSION_STATUS_RESTART;
1784 case -8969 /* codecBadDataErr */:
1785 case kVTVideoDecoderBadDataErr:
1787 *p_vtsession_status = VTSESSION_STATUS_ABORT;
1791 return VLC_EGENERIC;
1794 #pragma mark - actual decoding
1796 static void RequestFlush(decoder_t *p_dec)
1798 decoder_sys_t *p_sys = p_dec->p_sys;
1800 vlc_mutex_lock(&p_sys->lock);
1801 p_sys->b_vt_flush = true;
1802 vlc_mutex_unlock(&p_sys->lock);
1805 static void Drain(decoder_t *p_dec, bool flush)
1807 decoder_sys_t *p_sys = p_dec->p_sys;
1809 /* draining: return last pictures of the reordered queue */
1810 vlc_mutex_lock(&p_sys->lock);
1811 p_sys->b_vt_flush = true;
1812 DrainDPBLocked(p_dec, flush);
1813 vlc_mutex_unlock(&p_sys->lock);
1815 if (p_sys->session && p_sys->b_vt_feed)
1816 VTDecompressionSessionWaitForAsynchronousFrames(p_sys->session);
1818 vlc_mutex_lock(&p_sys->lock);
1819 assert(RemoveOneFrameFromDPB(p_sys) == NULL);
1820 p_sys->b_vt_flush = false;
1821 p_sys->b_vt_feed = false;
1822 p_sys->b_drop_blocks = false;
1823 vlc_mutex_unlock(&p_sys->lock);
1826 static int DecodeBlock(decoder_t *p_dec, block_t *p_block)
1828 decoder_sys_t *p_sys = p_dec->p_sys;
1830 if (p_sys->b_vt_flush)
1836 if (p_block == NULL)
1838 Drain(p_dec, false);
1839 return VLCDEC_SUCCESS;
1842 vlc_mutex_lock(&p_sys->lock);
1844 if (p_block->i_flags & BLOCK_FLAG_INTERLACED_MASK)
1846 #if TARGET_OS_IPHONE
1847 msg_Warn(p_dec, "VT decoder doesn't handle deinterlacing on iOS, "
1849 p_sys->vtsession_status = VTSESSION_STATUS_ABORT;
1851 if (!p_sys->b_cvpx_format_forced
1852 && p_sys->i_cvpx_format == kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange)
1854 /* In case of interlaced content, force VT to output I420 since our
1855 * SW deinterlacer handle this chroma natively. This avoids having
1856 * 2 extra conversions (CVPX->I420 then I420->CVPX). */
1858 p_sys->i_cvpx_format = kCVPixelFormatType_420YpCbCr8Planar;
1859 msg_Warn(p_dec, "Interlaced content: forcing VT to output I420");
1860 if (p_sys->session != nil && p_sys->vtsession_status == VTSESSION_STATUS_OK)
1862 msg_Warn(p_dec, "restarting vt session (color changed)");
1863 vlc_mutex_unlock(&p_sys->lock);
1865 /* Drain before stopping */
1866 Drain(p_dec, false);
1867 StopVideoToolbox(p_dec);
1869 vlc_mutex_lock(&p_sys->lock);
1875 if (p_sys->vtsession_status == VTSESSION_STATUS_RESTART ||
1876 p_sys->vtsession_status == VTSESSION_STATUS_RESTART_CHROMA)
1879 if (p_sys->vtsession_status == VTSESSION_STATUS_RESTART_CHROMA)
1881 if (p_sys->i_cvpx_format == 0 && p_sys->b_cvpx_format_forced)
1883 /* Already tried to fallback to the original chroma, aborting... */
1888 p_sys->i_cvpx_format = 0;
1889 p_sys->b_cvpx_format_forced = true;
1894 do_restart = p_sys->i_restart_count <= VT_RESTART_MAX;
1898 msg_Warn(p_dec, "restarting vt session (dec callback failed)");
1899 vlc_mutex_unlock(&p_sys->lock);
1901 /* Session will be started by Late Start code block */
1902 StopVideoToolbox(p_dec);
1903 if (p_dec->fmt_in.i_extra == 0)
1905 /* Clean old parameter sets since they may be corrupt */
1906 hxxx_helper_clean(&p_sys->hh);
1909 vlc_mutex_lock(&p_sys->lock);
1910 p_sys->vtsession_status = VTSESSION_STATUS_OK;
1914 msg_Warn(p_dec, "too many vt failure...");
1915 p_sys->vtsession_status = VTSESSION_STATUS_ABORT;
1919 if (p_sys->vtsession_status == VTSESSION_STATUS_ABORT)
1921 vlc_mutex_unlock(&p_sys->lock);
1923 msg_Err(p_dec, "decoder failure, Abort.");
1924 /* Add an empty variable so that videotoolbox won't be loaded again for
1926 var_Create(p_dec, "videotoolbox-failed", VLC_VAR_VOID);
1927 return VLCDEC_RELOAD;
1929 vlc_mutex_unlock(&p_sys->lock);
1931 if (unlikely(p_block->i_flags&(BLOCK_FLAG_CORRUPTED)))
1933 if (p_sys->b_vt_feed)
1935 Drain(p_dec, false);
1941 bool b_config_changed = false;
1942 if(p_sys->pf_process_block)
1944 p_block = p_sys->pf_process_block(p_dec, p_block, &b_config_changed);
1946 return VLCDEC_SUCCESS;
1949 frame_info_t *p_info = CreateReorderInfo(p_dec, p_block);
1950 if(unlikely(!p_info))
1953 if (!p_sys->session /* Late Start */||
1954 (b_config_changed && p_info->b_flush))
1956 if (p_sys->session &&
1957 p_sys->pf_need_restart &&
1958 p_sys->pf_need_restart(p_dec,p_sys->session))
1960 msg_Dbg(p_dec, "parameters sets changed: draining decoder");
1961 Drain(p_dec, false);
1962 msg_Dbg(p_dec, "parameters sets changed: restarting decoder");
1963 StopVideoToolbox(p_dec);
1968 if ((p_sys->pf_codec_supported && !p_sys->pf_codec_supported(p_dec))
1969 || StartVideoToolbox(p_dec) != VLC_SUCCESS)
1971 /* The current device doesn't handle the profile/level, abort */
1972 vlc_mutex_lock(&p_sys->lock);
1973 p_sys->vtsession_status = VTSESSION_STATUS_ABORT;
1974 vlc_mutex_unlock(&p_sys->lock);
1978 if (!p_sys->session) /* Start Failed */
1985 if (!p_sys->b_vt_feed && p_sys->b_vt_need_keyframe && !p_info->b_keyframe)
1991 CMSampleBufferRef sampleBuffer =
1992 VTSampleBufferCreate(p_dec, p_sys->videoFormatDescription, p_block);
1993 if (unlikely(!sampleBuffer))
1999 VTDecodeInfoFlags flagOut;
2000 VTDecodeFrameFlags decoderFlags = kVTDecodeFrame_EnableAsynchronousDecompression;
2003 VTDecompressionSessionDecodeFrame(p_sys->session, sampleBuffer,
2004 decoderFlags, p_info, &flagOut);
2006 enum vtsession_status vtsession_status;
2007 if (HandleVTStatus(p_dec, status, &vtsession_status) == VLC_SUCCESS)
2009 p_sys->b_vt_feed = true;
2010 if( p_block->i_flags & BLOCK_FLAG_END_OF_SEQUENCE )
2011 Drain( p_dec, false );
2015 vlc_mutex_lock(&p_sys->lock);
2016 if (vtsession_status == VTSESSION_STATUS_RESTART)
2017 p_sys->i_restart_count++;
2018 p_sys->vtsession_status = vtsession_status;
2019 /* In case of abort, the decoder module will be reloaded next time
2020 * since we already modified the input block */
2021 vlc_mutex_unlock(&p_sys->lock);
2023 CFRelease(sampleBuffer);
2026 block_Release(p_block);
2027 return VLCDEC_SUCCESS;
2030 static int UpdateVideoFormat(decoder_t *p_dec, CVPixelBufferRef imageBuffer)
2032 decoder_sys_t *p_sys = p_dec->p_sys;
2033 NSDictionary *attachmentDict =
2034 (__bridge NSDictionary *)CVBufferGetAttachments(imageBuffer, kCVAttachmentMode_ShouldPropagate);
2036 if (attachmentDict != nil && attachmentDict.count > 0
2037 && p_dec->fmt_out.video.chroma_location == CHROMA_LOCATION_UNDEF)
2039 NSString *chromaLocation = attachmentDict[(NSString *)kCVImageBufferChromaLocationTopFieldKey];
2040 if (chromaLocation != nil) {
2041 if ([chromaLocation isEqualToString:(NSString *)kCVImageBufferChromaLocation_Left] ||
2042 [chromaLocation isEqualToString:(NSString *)kCVImageBufferChromaLocation_DV420])
2043 p_dec->fmt_out.video.chroma_location = CHROMA_LOCATION_LEFT;
2044 else if ([chromaLocation isEqualToString:(NSString *)kCVImageBufferChromaLocation_Center])
2045 p_dec->fmt_out.video.chroma_location = CHROMA_LOCATION_CENTER;
2046 else if ([chromaLocation isEqualToString:(NSString *)kCVImageBufferChromaLocation_TopLeft])
2047 p_dec->fmt_out.video.chroma_location = CHROMA_LOCATION_TOP_LEFT;
2048 else if ([chromaLocation isEqualToString:(NSString *)kCVImageBufferChromaLocation_Top])
2049 p_dec->fmt_out.video.chroma_location = CHROMA_LOCATION_TOP_CENTER;
2051 if (p_dec->fmt_out.video.chroma_location == CHROMA_LOCATION_UNDEF)
2053 chromaLocation = attachmentDict[(NSString *)kCVImageBufferChromaLocationBottomFieldKey];
2054 if (chromaLocation != nil) {
2055 if ([chromaLocation isEqualToString:(NSString *)kCVImageBufferChromaLocation_BottomLeft])
2056 p_dec->fmt_out.video.chroma_location = CHROMA_LOCATION_BOTTOM_LEFT;
2057 else if ([chromaLocation isEqualToString:(NSString *)kCVImageBufferChromaLocation_Bottom])
2058 p_dec->fmt_out.video.chroma_location = CHROMA_LOCATION_BOTTOM_CENTER;
2063 uint32_t cvfmt = CVPixelBufferGetPixelFormatType(imageBuffer);
2064 msg_Info(p_dec, "vt cvpx chroma: %4.4s",
2065 (const char *)&(uint32_t) { htonl(cvfmt) });
2068 case kCVPixelFormatType_422YpCbCr8:
2070 p_dec->fmt_out.i_codec = VLC_CODEC_CVPX_UYVY;
2071 assert(CVPixelBufferIsPlanar(imageBuffer) == false);
2073 case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
2074 case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
2075 p_dec->fmt_out.i_codec = VLC_CODEC_CVPX_NV12;
2076 assert(CVPixelBufferIsPlanar(imageBuffer) == true);
2078 case 'xf20': /* kCVPixelFormatType_420YpCbCr10BiPlanarFullRange */
2079 case 'x420': /* kCVPixelFormatType_420YpCbCr10BiPlanarVideoRange */
2080 p_dec->fmt_out.i_codec = VLC_CODEC_CVPX_P010;
2081 assert(CVPixelBufferIsPlanar(imageBuffer) == true);
2083 case kCVPixelFormatType_420YpCbCr8Planar:
2084 p_dec->fmt_out.i_codec = VLC_CODEC_CVPX_I420;
2085 assert(CVPixelBufferIsPlanar(imageBuffer) == true);
2087 case kCVPixelFormatType_32BGRA:
2088 p_dec->fmt_out.i_codec = VLC_CODEC_CVPX_BGRA;
2089 assert(CVPixelBufferIsPlanar(imageBuffer) == false);
2092 p_sys->vtsession_status = VTSESSION_STATUS_ABORT;
2095 p_dec->fmt_out.video.i_chroma = p_dec->fmt_out.i_codec;
2096 if (decoder_UpdateVideoFormat(p_dec) != 0)
2098 p_sys->vtsession_status = VTSESSION_STATUS_ABORT;
2105 pic_holder_on_cvpx_released(CVPixelBufferRef cvpx, void *data, unsigned nb_fields)
2107 struct pic_holder *pic_holder = data;
2109 vlc_mutex_lock(&pic_holder->lock);
2110 assert((int) pic_holder->nb_field_out - nb_fields >= 0);
2111 pic_holder->nb_field_out -= nb_fields;
2112 if (pic_holder->nb_field_out == 0 && pic_holder->closed)
2114 vlc_mutex_unlock(&pic_holder->lock);
2115 pic_holder_clean(pic_holder);
2119 vlc_cond_broadcast(&pic_holder->wait);
2120 vlc_mutex_unlock(&pic_holder->lock);
2125 pic_holder_update_reorder_max(struct pic_holder *pic_holder, uint8_t pic_reorder_max,
2128 vlc_mutex_lock(&pic_holder->lock);
2130 pic_holder->field_reorder_max = pic_reorder_max * (nb_field < 2 ? 2 : nb_field);
2131 vlc_cond_signal(&pic_holder->wait);
2133 vlc_mutex_unlock(&pic_holder->lock);
2136 static int pic_holder_wait(struct pic_holder *pic_holder, const picture_t *pic)
2138 const uint8_t reserved_fields = 2 * (pic->i_nb_fields < 2 ? 2 : pic->i_nb_fields);
2140 vlc_mutex_lock(&pic_holder->lock);
2142 /* Wait 200 ms max. We can't really know what the video output will do with
2143 * output pictures (will they be rendered immediately ?), so don't wait
2144 * infinitely. The output will be paced anyway by the vlc_cond_timedwait()
2146 vlc_tick_t deadline = vlc_tick_now() + VLC_TICK_FROM_MS(200);
2148 while (ret == 0 && pic_holder->field_reorder_max != 0
2149 && pic_holder->nb_field_out >= pic_holder->field_reorder_max + reserved_fields)
2150 ret = vlc_cond_timedwait(&pic_holder->wait, &pic_holder->lock, deadline);
2151 pic_holder->nb_field_out += pic->i_nb_fields;
2153 vlc_mutex_unlock(&pic_holder->lock);
2158 static void DecoderCallback(void *decompressionOutputRefCon,
2159 void *sourceFrameRefCon,
2161 VTDecodeInfoFlags infoFlags,
2162 CVPixelBufferRef imageBuffer,
2166 VLC_UNUSED(duration);
2167 decoder_t *p_dec = (decoder_t *)decompressionOutputRefCon;
2168 decoder_sys_t *p_sys = p_dec->p_sys;
2169 frame_info_t *p_info = (frame_info_t *) sourceFrameRefCon;
2171 vlc_mutex_lock(&p_sys->lock);
2172 if (p_sys->b_vt_flush)
2175 enum vtsession_status vtsession_status;
2176 if (HandleVTStatus(p_dec, status, &vtsession_status) != VLC_SUCCESS)
2178 if (p_sys->vtsession_status != VTSESSION_STATUS_ABORT)
2180 p_sys->vtsession_status = vtsession_status;
2181 if (vtsession_status == VTSESSION_STATUS_RESTART)
2182 p_sys->i_restart_count++;
2186 if (unlikely(!imageBuffer))
2188 msg_Err(p_dec, "critical: null imageBuffer with a valid status");
2189 p_sys->vtsession_status = VTSESSION_STATUS_ABORT;
2193 if (p_sys->vtsession_status == VTSESSION_STATUS_ABORT)
2196 if (unlikely(!p_sys->b_format_propagated)) {
2197 p_sys->b_format_propagated =
2198 UpdateVideoFormat(p_dec, imageBuffer) == VLC_SUCCESS;
2200 if (!p_sys->b_format_propagated)
2202 assert(p_dec->fmt_out.i_codec != 0);
2205 if (infoFlags & kVTDecodeInfo_FrameDropped)
2207 /* We can't trust VT, some decoded frames can be marked as dropped */
2208 msg_Dbg(p_dec, "decoder dropped frame");
2211 if (!CMTIME_IS_VALID(pts))
2214 if (CVPixelBufferGetDataSize(imageBuffer) == 0)
2219 /* Unlock the mutex because decoder_NewPicture() is blocking. Indeed,
2220 * it can wait indefinitely when the input is paused. */
2222 vlc_mutex_unlock(&p_sys->lock);
2224 picture_t *p_pic = decoder_NewPicture(p_dec);
2227 vlc_mutex_lock(&p_sys->lock);
2231 p_info->p_picture = p_pic;
2233 p_pic->date = pts.value;
2234 p_pic->b_force = p_info->b_forced;
2235 p_pic->b_progressive = p_info->b_progressive;
2236 if(!p_pic->b_progressive)
2238 p_pic->i_nb_fields = p_info->i_num_ts;
2239 p_pic->b_top_field_first = p_info->b_top_field_first;
2242 if (cvpxpic_attach_with_cb(p_pic, imageBuffer, pic_holder_on_cvpx_released,
2243 p_sys->pic_holder) != VLC_SUCCESS)
2245 vlc_mutex_lock(&p_sys->lock);
2249 /* VT is not pacing frame allocation. If we are not fast enough to
2250 * render (release) the output pictures, the VT session can end up
2251 * allocating way too many frames. This can be problematic for 4K
2252 * 10bits. To fix this issue, we ensure that we don't have too many
2253 * output frames allocated by waiting for the vout to release them.
2255 * FIXME: A proper way to fix this issue is to allow decoder modules to
2256 * specify the dpb and having the vout re-allocating output frames when
2257 * this number changes. */
2258 if (pic_holder_wait(p_sys->pic_holder, p_pic))
2259 msg_Warn(p_dec, "pic_holder_wait timed out");
2262 vlc_mutex_lock(&p_sys->lock);
2264 if (p_sys->b_vt_flush)
2266 picture_Release(p_pic);
2270 p_sys->i_restart_count = 0;
2272 OnDecodedFrame( p_dec, p_info );
2278 vlc_mutex_unlock(&p_sys->lock);