modules/codec/videotoolbox.m

   1 /*****************************************************************************
   2  * videotoolbox.m: Video Toolbox decoder
   3  *****************************************************************************
   4  * Copyright © 2014-2015 VideoLabs SAS
   5  *
   6  * Authors: Felix Paul Kühne <fkuehne # videolan.org>
   7  *
   8  * This program is free software; you can redistribute it and/or modify it
   9  * under the terms of the GNU Lesser General Public License as published by
  10  * the Free Software Foundation; either version 2.1 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16  * GNU Lesser General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU Lesser General Public License
  19  * along with this program; if not, write to the Free Software Foundation,
  20  * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  21  *****************************************************************************/
  22
  23 #pragma mark preamble
  24
  25 #ifdef HAVE_CONFIG_H
  26 # import "config.h"
  27 #endif
  28
  29 #import <vlc_common.h>
  30 #import <vlc_plugin.h>
  31 #import <vlc_codec.h>
  32 #import "hxxx_helper.h"
  33 #import <vlc_bits.h>
  34 #import <vlc_boxes.h>
  35 #import "vt_utils.h"
  36 #import "../packetizer/h264_nal.h"
  37 #import "../packetizer/h264_slice.h"
  38 #import "../packetizer/hxxx_nal.h"
  39 #import "../packetizer/hxxx_sei.h"
  40
  41 #import <VideoToolbox/VideoToolbox.h>
  42 #import <VideoToolbox/VTErrors.h>
  43
  44 #import <Foundation/Foundation.h>
  45 #import <TargetConditionals.h>
  46
  47 #import <sys/types.h>
  48 #import <sys/sysctl.h>
  49 #import <mach/machine.h>
  50
  51 #if TARGET_OS_IPHONE
  52 #import <UIKit/UIKit.h>
  53
  54 /* support iOS SDKs < v9.1 */
  55 #ifndef CPUFAMILY_ARM_TWISTER
  56 #define CPUFAMILY_ARM_TWISTER 0x92fb37c8
  57 #endif
  58
  59 #endif
  60
  61 #pragma mark - module descriptor
  62
  63 static int OpenDecoder(vlc_object_t *);
  64 static void CloseDecoder(vlc_object_t *);
  65
  66 #if MAC_OS_X_VERSION_MIN_ALLOWED < 1090
  67 const CFStringRef kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder = CFSTR("EnableHardwareAcceleratedVideoDecoder");
  68 const CFStringRef kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder = CFSTR("RequireHardwareAcceleratedVideoDecoder");
  69 #endif
  70
  71 #define VT_REQUIRE_HW_DEC N_("Use Hardware decoders only")
  72 #define VT_TEMPO_DEINTERLACE N_("Deinterlacing")
  73 #define VT_TEMPO_DEINTERLACE_LONG N_("If interlaced content is detected, temporal deinterlacing is enabled at the expense of a pipeline delay.")
  74 #define VT_FORCE_CVPX_CHROMA "Force the VT decoder CVPX chroma"
  75 #define VT_FORCE_CVPX_CHROMA_LONG "Values can be 'BGRA', 'y420', '420f', '420v', '2vuy'. \
  76     By Default, the best chroma is choosen by the VT decoder."
  77
  78 vlc_module_begin()
  79 set_category(CAT_INPUT)
  80 set_subcategory(SUBCAT_INPUT_VCODEC)
  81 set_description(N_("VideoToolbox video decoder"))
  82 set_capability("video decoder",800)
  83 set_callbacks(OpenDecoder, CloseDecoder)
  84
  85 add_bool("videotoolbox-temporal-deinterlacing", true, VT_TEMPO_DEINTERLACE, VT_TEMPO_DEINTERLACE_LONG, false)
  86 add_bool("videotoolbox-hw-decoder-only", false, VT_REQUIRE_HW_DEC, VT_REQUIRE_HW_DEC, false)
  87 add_string("videotoolbox-cvpx-chroma", "", VT_FORCE_CVPX_CHROMA, VT_FORCE_CVPX_CHROMA_LONG, true);
  88 vlc_module_end()
  89
  90 #pragma mark - local prototypes
  91
  92 static int ESDSCreate(decoder_t *, uint8_t *, uint32_t);
  93 static int avcCFromAnnexBCreate(decoder_t *);
  94 static CFMutableDictionaryRef ExtradataInfoCreate(CFStringRef, void *, size_t);
  95 static int HandleVTStatus(decoder_t *, OSStatus);
  96 static int DecodeBlock(decoder_t *, block_t *);
  97 static void Flush(decoder_t *);
  98 static void DecoderCallback(void *, void *, OSStatus, VTDecodeInfoFlags,
  99                             CVPixelBufferRef, CMTime, CMTime);
 100 static BOOL deviceSupportsAdvancedProfiles();
 101 static BOOL deviceSupportsAdvancedLevels();
 102
 103 typedef struct frame_info_t frame_info_t;
 104
 105 struct frame_info_t
 106 {
 107     picture_t *p_picture;
 108     int i_poc;
 109     int i_foc;
 110     bool b_forced;
 111     bool b_flush;
 112     bool b_field;
 113     bool b_progressive;
 114     bool b_top_field_first;
 115     uint8_t i_num_ts;
 116     unsigned i_length;
 117     frame_info_t *p_next;
 118 };
 119
 120 #pragma mark - decoder structure
 121
 122 #define H264_MAX_DPB 16
 123
 124 struct decoder_sys_t
 125 {
 126     CMVideoCodecType            codec;
 127     struct                      hxxx_helper hh;
 128
 129     bool                        b_vt_feed;
 130     bool                        b_vt_flush;
 131     VTDecompressionSessionRef   session;
 132     CMVideoFormatDescriptionRef videoFormatDescription;
 133     CFMutableDictionaryRef      decoderConfiguration;
 134     CFMutableDictionaryRef      destinationPixelBufferAttributes;
 135     CFMutableDictionaryRef      extradataInfo;
 136
 137     vlc_mutex_t                 lock;
 138     frame_info_t               *p_pic_reorder;
 139     uint8_t                     i_pic_reorder;
 140     uint8_t                     i_pic_reorder_max;
 141     bool                        b_invalid_pic_reorder_max;
 142     bool                        b_poc_based_reorder;
 143     bool                        b_enable_temporal_processing;
 144     bool                        b_handle_deint;
 145
 146     bool                        b_format_propagated;
 147     bool                        b_abort;
 148     int                         i_forced_cvpx_format;
 149
 150     poc_context_t               pocctx;
 151     date_t                      pts;
 152 };
 153
 154 #pragma mark - start & stop
 155
 156 static void GetSPSPPS(uint8_t i_pps_id, void *priv,
 157                       const h264_sequence_parameter_set_t **pp_sps,
 158                       const h264_picture_parameter_set_t **pp_pps)
 159 {
 160     decoder_sys_t *p_sys = priv;
 161
 162     *pp_pps = p_sys->hh.h264.pps_list[i_pps_id].h264_pps;
 163     if(*pp_pps == NULL)
 164         *pp_sps = NULL;
 165     else
 166         *pp_sps = p_sys->hh.h264.sps_list[(*pp_pps)->i_sps_id].h264_sps;
 167 }
 168
 169 struct sei_callback_s
 170 {
 171     uint8_t i_pic_struct;
 172     const h264_sequence_parameter_set_t *p_sps;
 173 };
 174
 175 static bool ParseH264SEI(const hxxx_sei_data_t *p_sei_data, void *priv)
 176 {
 177
 178     if(p_sei_data->i_type == HXXX_SEI_PIC_TIMING)
 179     {
 180         struct sei_callback_s *s = priv;
 181         if(s->p_sps && s->p_sps->vui.b_valid)
 182         {
 183             if(s->p_sps->vui.b_hrd_parameters_present_flag)
 184             {
 185                 bs_read(p_sei_data->p_bs, s->p_sps->vui.i_cpb_removal_delay_length_minus1 + 1);
 186                 bs_read(p_sei_data->p_bs, s->p_sps->vui.i_dpb_output_delay_length_minus1 + 1);
 187             }
 188
 189             if(s->p_sps->vui.b_pic_struct_present_flag)
 190                s->i_pic_struct = bs_read( p_sei_data->p_bs, 4);
 191         }
 192         return false;
 193     }
 194
 195     return true;
 196 }
 197
 198 static bool ParseH264NAL(decoder_t *p_dec,
 199                          const uint8_t *p_buffer, size_t i_buffer,
 200                          uint8_t i_nal_length_size, frame_info_t *p_info)
 201 {
 202     decoder_sys_t *p_sys = p_dec->p_sys;
 203     hxxx_iterator_ctx_t itctx;
 204     hxxx_iterator_init(&itctx, p_buffer, i_buffer, i_nal_length_size);
 205
 206     const uint8_t *p_nal; size_t i_nal;
 207     const uint8_t *p_sei_nal = NULL; size_t i_sei_nal = 0;
 208     while(hxxx_iterate_next(&itctx, &p_nal, &i_nal))
 209     {
 210         if(i_nal < 2)
 211             continue;
 212
 213         const enum h264_nal_unit_type_e i_nal_type = p_nal[0] & 0x1F;
 214
 215         if (i_nal_type <= H264_NAL_SLICE_IDR && i_nal_type != H264_NAL_UNKNOWN)
 216         {
 217             h264_slice_t slice;
 218             if(!h264_decode_slice(p_nal, i_nal, GetSPSPPS, p_sys, &slice))
 219                 return false;
 220
 221             const h264_sequence_parameter_set_t *p_sps;
 222             const h264_picture_parameter_set_t *p_pps;
 223             GetSPSPPS(slice.i_pic_parameter_set_id, p_sys, &p_sps, &p_pps);
 224             if(p_sps)
 225             {
 226                 if(!p_sys->b_invalid_pic_reorder_max && i_nal_type == H264_NAL_SLICE_IDR)
 227                 {
 228                     unsigned dummy;
 229                     uint8_t i_reorder;
 230                     h264_get_dpb_values(p_sps, &i_reorder, &dummy);
 231                     vlc_mutex_lock(&p_sys->lock);
 232                     p_sys->i_pic_reorder_max = i_reorder;
 233                     vlc_mutex_unlock(&p_sys->lock);
 234                 }
 235
 236                 int bFOC;
 237                 h264_compute_poc(p_sps, &slice, &p_sys->pocctx,
 238                                  &p_info->i_poc, &p_info->i_foc, &bFOC);
 239
 240                 p_info->b_flush = (slice.type == H264_SLICE_TYPE_I) || slice.has_mmco5;
 241                 p_info->b_field = slice.i_field_pic_flag;
 242                 p_info->b_progressive = !p_sps->mb_adaptive_frame_field_flag &&
 243                                         !slice.i_field_pic_flag;
 244
 245                 struct sei_callback_s sei;
 246                 sei.p_sps = p_sps;
 247                 sei.i_pic_struct = UINT8_MAX;
 248
 249                 if(p_sei_nal)
 250                     HxxxParseSEI(p_sei_nal, i_sei_nal, 1, ParseH264SEI, &sei);
 251
 252                 p_info->i_num_ts = h264_get_num_ts(p_sps, &slice, sei.i_pic_struct,
 253                                                    p_info->i_foc, bFOC);
 254
 255                 if(!p_info->b_progressive)
 256                     p_info->b_top_field_first = (sei.i_pic_struct % 2 == 1);
 257
 258                 /* Set frame rate for timings in case of missing rate */
 259                 if( (!p_dec->fmt_in.video.i_frame_rate_base ||
 260                      !p_dec->fmt_in.video.i_frame_rate) &&
 261                     p_sps->vui.i_time_scale && p_sps->vui.i_num_units_in_tick )
 262                 {
 263                     date_Change( &p_sys->pts, p_sps->vui.i_time_scale,
 264                                               p_sps->vui.i_num_units_in_tick );
 265                 }
 266             }
 267
 268             return true; /* No need to parse further NAL */
 269         }
 270         else if(i_nal_type == H264_NAL_SEI)
 271         {
 272             p_sei_nal = p_nal;
 273             i_sei_nal = i_nal;
 274         }
 275     }
 276
 277     return false;
 278 }
 279
 280 static void InsertIntoDPB(decoder_sys_t *p_sys, frame_info_t *p_info)
 281 {
 282     frame_info_t **pp_lead_in = &p_sys->p_pic_reorder;
 283
 284     for( ;; pp_lead_in = & ((*pp_lead_in)->p_next))
 285     {
 286         bool b_insert;
 287         if(*pp_lead_in == NULL)
 288             b_insert = true;
 289         else if(p_sys->b_poc_based_reorder)
 290             b_insert = ((*pp_lead_in)->i_foc > p_info->i_foc);
 291         else
 292             b_insert = ((*pp_lead_in)->p_picture->date >= p_info->p_picture->date);
 293
 294         if(b_insert)
 295         {
 296             p_info->p_next = *pp_lead_in;
 297             *pp_lead_in = p_info;
 298             p_sys->i_pic_reorder += (p_info->b_field) ? 1 : 2;
 299             break;
 300         }
 301     }
 302 #if 0
 303     for(frame_info_t *p_in=p_sys->p_pic_reorder; p_in; p_in = p_in->p_next)
 304         printf(" %d", p_in->i_foc);
 305     printf("\n");
 306 #endif
 307 }
 308
 309 static picture_t * RemoveOneFrameFromDPB(decoder_sys_t *p_sys)
 310 {
 311     frame_info_t *p_info = p_sys->p_pic_reorder;
 312     if(p_info == NULL)
 313         return NULL;
 314
 315     const int i_framepoc = p_info->i_poc;
 316
 317     picture_t *p_ret = NULL;
 318     picture_t **pp_ret_last = &p_ret;
 319     bool b_dequeue;
 320
 321     do
 322     {
 323         picture_t *p_field = p_info->p_picture;
 324
 325         /* Compute time if missing */
 326         if (p_field->date == VLC_TS_INVALID)
 327             p_field->date = date_Get(&p_sys->pts);
 328         else
 329             date_Set(&p_sys->pts, p_field->date);
 330
 331         /* Set next picture time, in case it is missing */
 332         if (p_info->i_length)
 333             date_Set(&p_sys->pts, p_field->date + p_info->i_length);
 334         else
 335             date_Increment(&p_sys->pts, p_info->i_num_ts);
 336
 337         *pp_ret_last = p_field;
 338         pp_ret_last = &p_field->p_next;
 339
 340         p_sys->i_pic_reorder -= (p_info->b_field) ? 1 : 2;
 341
 342         p_sys->p_pic_reorder = p_info->p_next;
 343         free(p_info);
 344         p_info = p_sys->p_pic_reorder;
 345
 346         if (p_info)
 347         {
 348             if (p_sys->b_poc_based_reorder)
 349                 b_dequeue = (p_info->i_poc == i_framepoc);
 350             else
 351                 b_dequeue = (p_field->date == p_info->p_picture->date);
 352         }
 353         else b_dequeue = false;
 354
 355     } while(b_dequeue);
 356
 357     return p_ret;
 358 }
 359
 360 static void DrainDPB(decoder_t *p_dec)
 361 {
 362     decoder_sys_t *p_sys = p_dec->p_sys;
 363     for( ;; )
 364     {
 365         picture_t *p_fields = RemoveOneFrameFromDPB(p_sys);
 366         if (p_fields == NULL)
 367             break;
 368         do
 369         {
 370             picture_t *p_next = p_fields->p_next;
 371             p_fields->p_next = NULL;
 372             decoder_QueueVideo(p_dec, p_fields);
 373             p_fields = p_next;
 374         } while(p_fields != NULL);
 375     }
 376 }
 377
 378 static frame_info_t * CreateReorderInfo(decoder_t *p_dec, const block_t *p_block)
 379 {
 380     decoder_sys_t *p_sys = p_dec->p_sys;
 381     frame_info_t *p_info = calloc(1, sizeof(*p_info));
 382     if (!p_info)
 383         return NULL;
 384
 385     if (p_sys->b_poc_based_reorder)
 386     {
 387         if(p_sys->codec != kCMVideoCodecType_H264 ||
 388            !ParseH264NAL(p_dec, p_block->p_buffer, p_block->i_buffer, 4, p_info))
 389         {
 390             assert(p_sys->codec == kCMVideoCodecType_H264);
 391             free(p_info);
 392             return NULL;
 393         }
 394     }
 395     else
 396     {
 397         p_info->i_num_ts = 2;
 398         p_info->b_progressive = true;
 399         p_info->b_field = false;
 400     }
 401
 402     p_info->i_length = p_block->i_length;
 403
 404     /* required for still pictures/menus */
 405     p_info->b_forced = (p_block->i_flags & BLOCK_FLAG_END_OF_SEQUENCE);
 406
 407     if (date_Get(&p_sys->pts) == VLC_TS_INVALID)
 408         date_Set(&p_sys->pts, p_block->i_dts);
 409
 410     return p_info;
 411 }
 412
 413 static void OnDecodedFrame(decoder_t *p_dec, frame_info_t *p_info)
 414 {
 415     decoder_sys_t *p_sys = p_dec->p_sys;
 416     assert(p_info->p_picture);
 417     while(p_info->b_flush || p_sys->i_pic_reorder >= (p_sys->i_pic_reorder_max * 2))
 418     {
 419         /* First check if DPB sizing was correct before removing one frame */
 420         if (p_sys->p_pic_reorder && !p_info->b_flush &&
 421             p_sys->i_pic_reorder_max < H264_MAX_DPB)
 422         {
 423             if(p_sys->b_poc_based_reorder && p_sys->p_pic_reorder->i_foc > p_info->i_foc)
 424             {
 425                 p_sys->b_invalid_pic_reorder_max = true;
 426                 p_sys->i_pic_reorder_max++;
 427                 msg_Info(p_dec, "Raising max DPB to %"PRIu8, p_sys->i_pic_reorder_max);
 428                 break;
 429             }
 430             else if (!p_sys->b_poc_based_reorder &&
 431                      p_info->p_picture->date > VLC_TS_INVALID &&
 432                      p_sys->p_pic_reorder->p_picture->date > p_info->p_picture->date)
 433             {
 434                 p_sys->b_invalid_pic_reorder_max = true;
 435                 p_sys->i_pic_reorder_max++;
 436                 msg_Info(p_dec, "Raising max DPB to %"PRIu8, p_sys->i_pic_reorder_max);
 437                 break;
 438             }
 439         }
 440
 441         picture_t *p_fields = RemoveOneFrameFromDPB(p_sys);
 442         if (p_fields == NULL)
 443             break;
 444         do
 445         {
 446             picture_t *p_next = p_fields->p_next;
 447             p_fields->p_next = NULL;
 448             decoder_QueueVideo(p_dec, p_fields);
 449             p_fields = p_next;
 450         } while(p_fields != NULL);
 451     }
 452
 453     InsertIntoDPB(p_sys, p_info);
 454 }
 455
 456 static CMVideoCodecType CodecPrecheck(decoder_t *p_dec)
 457 {
 458     uint8_t i_profile = 0xFF, i_level = 0xFF;
 459     bool b_ret = false;
 460     CMVideoCodecType codec;
 461
 462     /* check for the codec we can and want to decode */
 463     switch (p_dec->fmt_in.i_codec) {
 464         case VLC_CODEC_H264:
 465             codec = kCMVideoCodecType_H264;
 466
 467             b_ret = h264_get_profile_level(&p_dec->fmt_in, &i_profile, &i_level, NULL);
 468             if (!b_ret) {
 469                 msg_Warn(p_dec, "H264 profile and level parsing failed because it didn't arrive yet");
 470                 return kCMVideoCodecType_H264;
 471             }
 472
 473             msg_Dbg(p_dec, "trying to decode MPEG-4 Part 10: profile %" PRIx8 ", level %" PRIx8, i_profile, i_level);
 474
 475             switch (i_profile) {
 476                 case PROFILE_H264_BASELINE:
 477                 case PROFILE_H264_MAIN:
 478                 case PROFILE_H264_HIGH:
 479                     break;
 480
 481                 case PROFILE_H264_HIGH_10:
 482                 {
 483                     if (deviceSupportsAdvancedProfiles())
 484                         break;
 485                 }
 486
 487                 default:
 488                 {
 489                     msg_Dbg(p_dec, "unsupported H264 profile %" PRIx8, i_profile);
 490                     return -1;
 491                 }
 492             }
 493
 494 #if !TARGET_OS_IPHONE
 495             /* a level higher than 5.2 was not tested, so don't dare to
 496              * try to decode it*/
 497             if (i_level > 52) {
 498                 msg_Dbg(p_dec, "unsupported H264 level %" PRIx8, i_level);
 499                 return -1;
 500             }
 501 #else
 502             /* on SoC A8, 4.2 is the highest specified profile */
 503             if (i_level > 42) {
 504                 /* on Twister, we can do up to 5.2 */
 505                 if (!deviceSupportsAdvancedLevels() || i_level > 52) {
 506                     msg_Dbg(p_dec, "unsupported H264 level %" PRIx8, i_level);
 507                     return -1;
 508                 }
 509             }
 510 #endif
 511
 512             break;
 513         case VLC_CODEC_MP4V:
 514         {
 515             if (p_dec->fmt_in.i_original_fourcc == VLC_FOURCC( 'X','V','I','D' )) {
 516                 msg_Warn(p_dec, "XVID decoding not implemented, fallback on software");
 517                 return -1;
 518             }
 519
 520             msg_Dbg(p_dec, "Will decode MP4V with original FourCC '%4.4s'", (char *)&p_dec->fmt_in.i_original_fourcc);
 521             codec = kCMVideoCodecType_MPEG4Video;
 522
 523             break;
 524         }
 525 #if !TARGET_OS_IPHONE
 526         case VLC_CODEC_H263:
 527             codec = kCMVideoCodecType_H263;
 528             break;
 529
 530             /* there are no DV or ProRes decoders on iOS, so bailout early */
 531         case VLC_CODEC_PRORES:
 532             /* the VT decoder can't differenciate between the ProRes flavors, so we do it */
 533             switch (p_dec->fmt_in.i_original_fourcc) {
 534                 case VLC_FOURCC( 'a','p','4','c' ):
 535                 case VLC_FOURCC( 'a','p','4','h' ):
 536                     codec = kCMVideoCodecType_AppleProRes4444;
 537                     break;
 538
 539                 case VLC_FOURCC( 'a','p','c','h' ):
 540                     codec = kCMVideoCodecType_AppleProRes422HQ;
 541                     break;
 542
 543                 case VLC_FOURCC( 'a','p','c','s' ):
 544                     codec = kCMVideoCodecType_AppleProRes422LT;
 545                     break;
 546
 547                 case VLC_FOURCC( 'a','p','c','o' ):
 548                     codec = kCMVideoCodecType_AppleProRes422Proxy;
 549                     break;
 550
 551                 default:
 552                     codec = kCMVideoCodecType_AppleProRes422;
 553                     break;
 554             }
 555             if (codec != 0)
 556                 break;
 557
 558         case VLC_CODEC_DV:
 559             /* the VT decoder can't differenciate between PAL and NTSC, so we need to do it */
 560             switch (p_dec->fmt_in.i_original_fourcc) {
 561                 case VLC_FOURCC( 'd', 'v', 'c', ' '):
 562                 case VLC_FOURCC( 'd', 'v', ' ', ' '):
 563                     msg_Dbg(p_dec, "Decoding DV NTSC");
 564                     codec = kCMVideoCodecType_DVCNTSC;
 565                     break;
 566
 567                 case VLC_FOURCC( 'd', 'v', 's', 'd'):
 568                 case VLC_FOURCC( 'd', 'v', 'c', 'p'):
 569                 case VLC_FOURCC( 'D', 'V', 'S', 'D'):
 570                     msg_Dbg(p_dec, "Decoding DV PAL");
 571                     codec = kCMVideoCodecType_DVCPAL;
 572                     break;
 573
 574                 default:
 575                     break;
 576             }
 577             if (codec != 0)
 578                 break;
 579 #endif
 580             /* mpgv / mp2v needs fixing, so disable it for now */
 581 #if 0
 582         case VLC_CODEC_MPGV:
 583             codec = kCMVideoCodecType_MPEG1Video;
 584             break;
 585         case VLC_CODEC_MP2V:
 586             codec = kCMVideoCodecType_MPEG2Video;
 587             break;
 588 #endif
 589
 590         default:
 591 #ifndef NDEBUG
 592             msg_Err(p_dec, "'%4.4s' is not supported", (char *)&p_dec->fmt_in.i_codec);
 593 #endif
 594             return -1;
 595     }
 596
 597     return codec;
 598 }
 599
 600 static int StartVideoToolbox(decoder_t *p_dec)
 601 {
 602     decoder_sys_t *p_sys = p_dec->p_sys;
 603     OSStatus status;
 604
 605     assert(p_sys->extradataInfo != nil);
 606
 607     p_sys->decoderConfiguration = cfdict_create(2);
 608     if (p_sys->decoderConfiguration == NULL)
 609         return VLC_ENOMEM;
 610
 611     CFDictionarySetValue(p_sys->decoderConfiguration,
 612                          kCVImageBufferChromaLocationBottomFieldKey,
 613                          kCVImageBufferChromaLocation_Left);
 614     CFDictionarySetValue(p_sys->decoderConfiguration,
 615                          kCVImageBufferChromaLocationTopFieldKey,
 616                          kCVImageBufferChromaLocation_Left);
 617
 618     CFDictionarySetValue(p_sys->decoderConfiguration,
 619                          kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms,
 620                          p_sys->extradataInfo);
 621
 622     /* pixel aspect ratio */
 623     CFMutableDictionaryRef pixelaspectratio = cfdict_create(2);
 624
 625     const unsigned i_video_width = p_dec->fmt_out.video.i_width;
 626     const unsigned i_video_height = p_dec->fmt_out.video.i_height;
 627     const unsigned i_sar_num = p_dec->fmt_out.video.i_sar_num;
 628     const unsigned i_sar_den = p_dec->fmt_out.video.i_sar_den;
 629
 630     if( p_dec->fmt_in.video.i_frame_rate_base && p_dec->fmt_in.video.i_frame_rate )
 631     {
 632         date_Init( &p_sys->pts, p_dec->fmt_in.video.i_frame_rate * 2,
 633                                 p_dec->fmt_in.video.i_frame_rate_base );
 634     }
 635     else date_Init( &p_sys->pts, 2 * 30000, 1001 );
 636
 637     cfdict_set_int32(pixelaspectratio,
 638                      kCVImageBufferPixelAspectRatioHorizontalSpacingKey,
 639                      i_sar_num);
 640     cfdict_set_int32(pixelaspectratio,
 641                      kCVImageBufferPixelAspectRatioVerticalSpacingKey,
 642                      i_sar_den);
 643     CFDictionarySetValue(p_sys->decoderConfiguration,
 644                          kCVImageBufferPixelAspectRatioKey,
 645                          pixelaspectratio);
 646     CFRelease(pixelaspectratio);
 647
 648     /* enable HW accelerated playback, since this is optional on OS X
 649      * note that the backend may still fallback on software mode if no
 650      * suitable hardware is available */
 651     CFDictionarySetValue(p_sys->decoderConfiguration,
 652                          kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder,
 653                          kCFBooleanTrue);
 654
 655     /* on OS X, we can force VT to fail if no suitable HW decoder is available,
 656      * preventing the aforementioned SW fallback */
 657     if (var_InheritInteger(p_dec, "videotoolbox-hw-decoder-only"))
 658         CFDictionarySetValue(p_sys->decoderConfiguration,
 659                              kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder,
 660                              kCFBooleanTrue);
 661
 662     CFDictionarySetValue(p_sys->decoderConfiguration,
 663                          kVTDecompressionPropertyKey_FieldMode,
 664                          kVTDecompressionProperty_FieldMode_DeinterlaceFields);
 665     CFDictionarySetValue(p_sys->decoderConfiguration,
 666                          kVTDecompressionPropertyKey_DeinterlaceMode,
 667                          kVTDecompressionProperty_DeinterlaceMode_Temporal);
 668
 669     /* create video format description */
 670     status = CMVideoFormatDescriptionCreate(kCFAllocatorDefault,
 671                                             p_sys->codec,
 672                                             i_video_width,
 673                                             i_video_height,
 674                                             p_sys->decoderConfiguration,
 675                                             &p_sys->videoFormatDescription);
 676     if (status) {
 677         CFRelease(p_sys->decoderConfiguration);
 678         msg_Err(p_dec, "video format description creation failed (%i)", (int)status);
 679         return VLC_EGENERIC;
 680     }
 681
 682     /* destination pixel buffer attributes */
 683     p_sys->destinationPixelBufferAttributes = cfdict_create(2);
 684
 685 #if !TARGET_OS_IPHONE
 686     CFDictionarySetValue(p_sys->destinationPixelBufferAttributes,
 687                          kCVPixelBufferIOSurfaceOpenGLTextureCompatibilityKey,
 688                          kCFBooleanTrue);
 689 #else
 690     CFDictionarySetValue(p_sys->destinationPixelBufferAttributes,
 691                          kCVPixelBufferOpenGLESCompatibilityKey,
 692                          kCFBooleanTrue);
 693 #endif
 694
 695     cfdict_set_int32(p_sys->destinationPixelBufferAttributes,
 696                      kCVPixelBufferWidthKey, i_video_width);
 697     cfdict_set_int32(p_sys->destinationPixelBufferAttributes,
 698                      kCVPixelBufferHeightKey, i_video_height);
 699
 700     if (p_sys->i_forced_cvpx_format != 0)
 701     {
 702         msg_Warn(p_dec, "forcing CVPX format: %4.4s",
 703                  (const char *) &p_sys->i_forced_cvpx_format);
 704         cfdict_set_int32(p_sys->destinationPixelBufferAttributes,
 705                          kCVPixelBufferPixelFormatTypeKey,
 706                          ntohl(p_sys->i_forced_cvpx_format));
 707     }
 708
 709     cfdict_set_int32(p_sys->destinationPixelBufferAttributes,
 710                      kCVPixelBufferBytesPerRowAlignmentKey,
 711                      i_video_width * 2);
 712
 713     /* setup decoder callback record */
 714     VTDecompressionOutputCallbackRecord decoderCallbackRecord;
 715     decoderCallbackRecord.decompressionOutputCallback = DecoderCallback;
 716     decoderCallbackRecord.decompressionOutputRefCon = p_dec;
 717
 718     /* create decompression session */
 719     status = VTDecompressionSessionCreate(kCFAllocatorDefault,
 720                                           p_sys->videoFormatDescription,
 721                                           p_sys->decoderConfiguration,
 722                                           p_sys->destinationPixelBufferAttributes,
 723                                           &decoderCallbackRecord, &p_sys->session);
 724
 725     if (HandleVTStatus(p_dec, status) != VLC_SUCCESS)
 726         return VLC_EGENERIC;
 727
 728     /* Check if the current session supports deinterlacing and temporal
 729      * processing */
 730     CFDictionaryRef supportedProps = NULL;
 731     status = VTSessionCopySupportedPropertyDictionary(p_sys->session,
 732                                                       &supportedProps);
 733     p_sys->b_handle_deint = status == noErr &&
 734         CFDictionaryContainsKey(supportedProps,
 735                                 kVTDecompressionPropertyKey_FieldMode);
 736     p_sys->b_enable_temporal_processing = status == noErr &&
 737         CFDictionaryContainsKey(supportedProps,
 738                                 kVTDecompressionProperty_DeinterlaceMode_Temporal);
 739     if (!p_sys->b_handle_deint)
 740         msg_Warn(p_dec, "VT decoder doesn't handle deinterlacing");
 741
 742     if (status == noErr)
 743         CFRelease(supportedProps);
 744
 745     return VLC_SUCCESS;
 746 }
 747
 748 static void StopVideoToolbox(decoder_t *p_dec, bool b_reset_format)
 749 {
 750     decoder_sys_t *p_sys = p_dec->p_sys;
 751
 752     if (p_sys->session != nil)
 753     {
 754         VTDecompressionSessionInvalidate(p_sys->session);
 755         CFRelease(p_sys->session);
 756         p_sys->session = nil;
 757
 758         if (b_reset_format)
 759         {
 760             p_sys->b_format_propagated = false;
 761             p_dec->fmt_out.i_codec = 0;
 762         }
 763         DrainDPB( p_dec );
 764     }
 765
 766     if (p_sys->videoFormatDescription != nil) {
 767         CFRelease(p_sys->videoFormatDescription);
 768         p_sys->videoFormatDescription = nil;
 769     }
 770     if (p_sys->decoderConfiguration != nil) {
 771         CFRelease(p_sys->decoderConfiguration);
 772         p_sys->decoderConfiguration = nil;
 773     }
 774     if (p_sys->destinationPixelBufferAttributes != nil) {
 775         CFRelease(p_sys->destinationPixelBufferAttributes);
 776         p_sys->destinationPixelBufferAttributes = nil;
 777     }
 778 }
 779
 780 static int RestartVideoToolbox(decoder_t *p_dec, bool b_reset_format)
 781 {
 782     decoder_sys_t *p_sys = p_dec->p_sys;
 783
 784     msg_Dbg(p_dec, "Restarting decoder session");
 785
 786     if (p_sys->session != nil)
 787         StopVideoToolbox(p_dec, b_reset_format);
 788
 789     return StartVideoToolbox(p_dec);
 790 }
 791
 792 #pragma mark - module open and close
 793
 794 static int SetupDecoderExtradata(decoder_t *p_dec)
 795 {
 796     decoder_sys_t *p_sys = p_dec->p_sys;
 797     CFMutableDictionaryRef extradata_info = NULL;
 798
 799     if (p_sys->codec == kCMVideoCodecType_H264)
 800     {
 801         hxxx_helper_init(&p_sys->hh, VLC_OBJECT(p_dec),
 802                          p_dec->fmt_in.i_codec, true);
 803         int i_ret = hxxx_helper_set_extra(&p_sys->hh, p_dec->fmt_in.p_extra,
 804                                           p_dec->fmt_in.i_extra);
 805         if (i_ret != VLC_SUCCESS)
 806             return i_ret;
 807         assert(p_sys->hh.pf_process_block != NULL);
 808
 809         if (p_dec->fmt_in.p_extra)
 810         {
 811             p_sys->extradataInfo = ExtradataInfoCreate(CFSTR("avcC"),
 812                                             p_dec->fmt_in.p_extra,
 813                                             p_dec->fmt_in.i_extra);
 814             if (p_sys->extradataInfo == nil)
 815                 return VLC_EGENERIC;
 816         }
 817         /* else: AnnexB case, we'll get extradata from first input blocks */
 818     }
 819     else if (p_sys->codec == kCMVideoCodecType_MPEG4Video)
 820     {
 821         if (!p_dec->fmt_in.i_extra)
 822             return VLC_EGENERIC;
 823         int i_ret = ESDSCreate(p_dec, p_dec->fmt_in.p_extra, p_dec->fmt_in.i_extra);
 824         if (i_ret != VLC_SUCCESS)
 825             return i_ret;
 826     }
 827     else
 828     {
 829         p_sys->extradataInfo = ExtradataInfoCreate(NULL, NULL, 0);
 830         if (p_sys->extradataInfo == nil)
 831             return VLC_EGENERIC;
 832     }
 833
 834     return VLC_SUCCESS;
 835 }
 836
 837 static int OpenDecoder(vlc_object_t *p_this)
 838 {
 839     decoder_t *p_dec = (decoder_t *)p_this;
 840
 841 #if TARGET_OS_IPHONE
 842     if (unlikely([[UIDevice currentDevice].systemVersion floatValue] < 8.0)) {
 843         msg_Warn(p_dec, "decoder skipped as OS is too old");
 844         return VLC_EGENERIC;
 845     }
 846 #endif
 847
 848     /* Fail if this module already failed to decode this ES */
 849     if (var_Type(p_dec, "videotoolbox-failed") != 0)
 850         return VLC_EGENERIC;
 851
 852     /* check quickly if we can digest the offered data */
 853     CMVideoCodecType codec;
 854     codec = CodecPrecheck(p_dec);
 855     if (codec == -1)
 856         return VLC_EGENERIC;
 857
 858     /* now that we see a chance to decode anything, allocate the
 859      * internals and start the decoding session */
 860     decoder_sys_t *p_sys;
 861     p_sys = malloc(sizeof(*p_sys));
 862     if (!p_sys)
 863         return VLC_ENOMEM;
 864     p_dec->p_sys = p_sys;
 865     p_sys->session = nil;
 866     p_sys->b_vt_feed = false;
 867     p_sys->b_vt_flush = false;
 868     p_sys->codec = codec;
 869     p_sys->videoFormatDescription = nil;
 870     p_sys->decoderConfiguration = nil;
 871     p_sys->destinationPixelBufferAttributes = nil;
 872     p_sys->extradataInfo = nil;
 873     p_sys->p_pic_reorder = NULL;
 874     p_sys->i_pic_reorder = 0;
 875     p_sys->i_pic_reorder_max = 4;
 876     p_sys->b_invalid_pic_reorder_max = false;
 877     p_sys->b_poc_based_reorder = false;
 878     p_sys->b_format_propagated = false;
 879     p_sys->b_abort = false;
 880     p_sys->b_enable_temporal_processing =
 881         var_InheritBool(p_dec, "videotoolbox-temporal-deinterlacing");
 882
 883     p_sys->i_forced_cvpx_format = 0;
 884     char *cvpx_chroma = var_InheritString(p_dec, "videotoolbox-cvpx-chroma");
 885     if (cvpx_chroma != NULL)
 886     {
 887         if (strlen(cvpx_chroma) != 4)
 888         {
 889             msg_Err(p_dec, "invalid videotoolbox-cvpx-chroma option");
 890             free(cvpx_chroma);
 891             free(p_sys);
 892             return VLC_EGENERIC;
 893         }
 894         memcpy(&p_sys->i_forced_cvpx_format, cvpx_chroma, 4);
 895         free(cvpx_chroma);
 896     }
 897     h264_poc_context_init( &p_sys->pocctx );
 898     vlc_mutex_init(&p_sys->lock);
 899
 900     /* return our proper VLC internal state */
 901     p_dec->fmt_out.video = p_dec->fmt_in.video;
 902     if (!p_dec->fmt_out.video.i_sar_num || !p_dec->fmt_out.video.i_sar_den)
 903     {
 904         p_dec->fmt_out.video.i_sar_num = 1;
 905         p_dec->fmt_out.video.i_sar_den = 1;
 906     }
 907     if (!p_dec->fmt_out.video.i_visible_width
 908      || !p_dec->fmt_out.video.i_visible_height)
 909     {
 910         p_dec->fmt_out.video.i_visible_width = p_dec->fmt_out.video.i_width;
 911         p_dec->fmt_out.video.i_visible_height = p_dec->fmt_out.video.i_height;
 912     }
 913     else
 914     {
 915         p_dec->fmt_out.video.i_width = p_dec->fmt_out.video.i_visible_width;
 916         p_dec->fmt_out.video.i_height = p_dec->fmt_out.video.i_visible_height;
 917     }
 918
 919     p_dec->fmt_out.i_codec = 0;
 920
 921     if( codec == kCMVideoCodecType_H264 )
 922         p_sys->b_poc_based_reorder = true;
 923
 924     int i_ret = SetupDecoderExtradata(p_dec);
 925     if (i_ret != VLC_SUCCESS)
 926         goto error;
 927
 928     if (p_sys->extradataInfo != nil)
 929     {
 930         i_ret = StartVideoToolbox(p_dec);
 931         if (i_ret != VLC_SUCCESS)
 932             goto error;
 933     } /* else: late opening */
 934
 935     p_dec->pf_decode = DecodeBlock;
 936     p_dec->pf_flush  = Flush;
 937
 938     msg_Info(p_dec, "Using Video Toolbox to decode '%4.4s'", (char *)&p_dec->fmt_in.i_codec);
 939
 940     return VLC_SUCCESS;
 941
 942 error:
 943     CloseDecoder(p_this);
 944     return i_ret;
 945 }
 946
 947 static void CloseDecoder(vlc_object_t *p_this)
 948 {
 949     decoder_t *p_dec = (decoder_t *)p_this;
 950     decoder_sys_t *p_sys = p_dec->p_sys;
 951
 952     StopVideoToolbox(p_dec, true);
 953
 954     if (p_sys->codec == kCMVideoCodecType_H264)
 955         hxxx_helper_clean(&p_sys->hh);
 956
 957     vlc_mutex_destroy(&p_sys->lock);
 958     free(p_sys);
 959 }
 960
 961 #pragma mark - helpers
 962
 963 static BOOL deviceSupportsAdvancedProfiles()
 964 {
 965 #if TARGET_IPHONE_SIMULATOR
 966     return NO;
 967 #endif
 968 #if TARGET_OS_IPHONE
 969     size_t size;
 970     cpu_type_t type;
 971
 972     size = sizeof(type);
 973     sysctlbyname("hw.cputype", &type, &size, NULL, 0);
 974
 975     /* Support for H264 profile HIGH 10 was introduced with the first 64bit Apple ARM SoC, the A7 */
 976     if (type == CPU_TYPE_ARM64)
 977         return YES;
 978
 979     return NO;
 980 #else
 981     return NO;
 982 #endif
 983 }
 984
 985 static BOOL deviceSupportsAdvancedLevels()
 986 {
 987 #if TARGET_IPHONE_SIMULATOR
 988     return YES;
 989 #endif
 990 #if TARGET_OS_IPHONE
 991     size_t size;
 992     int32_t cpufamily;
 993
 994     size = sizeof(cpufamily);
 995     sysctlbyname("hw.cpufamily", &cpufamily, &size, NULL, 0);
 996
 997     /* Proper 4K decoding requires a Twister SoC
 998      * Everything below will kill the decoder daemon */
 999     if (cpufamily == CPUFAMILY_ARM_TWISTER) {
1000         return YES;
1001     }
1002
1003     return NO;
1004 #else
1005     return YES;
1006 #endif
1007 }
1008
1009 static inline void bo_add_mp4_tag_descr(bo_t *p_bo, uint8_t tag, uint32_t size)
1010 {
1011     bo_add_8(p_bo, tag);
1012     for (int i = 3; i>0; i--)
1013         bo_add_8(p_bo, (size>>(7*i)) | 0x80);
1014     bo_add_8(p_bo, size & 0x7F);
1015 }
1016
1017 static int ESDSCreate(decoder_t *p_dec, uint8_t *p_buf, uint32_t i_buf_size)
1018 {
1019     decoder_sys_t *p_sys = p_dec->p_sys;
1020
1021     int full_size = 3 + 5 +13 + 5 + i_buf_size + 3;
1022     int config_size = 13 + 5 + i_buf_size;
1023     int padding = 12;
1024
1025     bo_t bo;
1026     bool status = bo_init(&bo, 1024);
1027     if (status != true)
1028         return VLC_EGENERIC;
1029
1030     bo_add_8(&bo, 0);       // Version
1031     bo_add_24be(&bo, 0);    // Flags
1032
1033     // elementary stream description tag
1034     bo_add_mp4_tag_descr(&bo, 0x03, full_size);
1035     bo_add_16be(&bo, 0);    // esid
1036     bo_add_8(&bo, 0);       // stream priority (0-3)
1037
1038     // decoder configuration description tag
1039     bo_add_mp4_tag_descr(&bo, 0x04, config_size);
1040     bo_add_8(&bo, 32);      // object type identification (32 == MPEG4)
1041     bo_add_8(&bo, 0x11);    // stream type
1042     bo_add_24be(&bo, 0);    // buffer size
1043     bo_add_32be(&bo, 0);    // max bitrate
1044     bo_add_32be(&bo, 0);    // avg bitrate
1045
1046     // decoder specific description tag
1047     bo_add_mp4_tag_descr(&bo, 0x05, i_buf_size);
1048     bo_add_mem(&bo, i_buf_size, p_buf);
1049
1050     // sync layer configuration description tag
1051     bo_add_8(&bo, 0x06);    // tag
1052     bo_add_8(&bo, 0x01);    // length
1053     bo_add_8(&bo, 0x02);    // no SL
1054
1055     p_sys->extradataInfo = ExtradataInfoCreate(CFSTR("esds"),
1056                                                bo.b->p_buffer, bo.b->i_buffer);
1057     bo_deinit(&bo);
1058
1059     return (p_sys->extradataInfo == nil) ? VLC_EGENERIC: VLC_SUCCESS;
1060 }
1061
1062 static int avcCFromAnnexBCreate(decoder_t *p_dec)
1063 {
1064     decoder_sys_t *p_sys = p_dec->p_sys;
1065
1066     if (p_sys->hh.h264.i_sps_count == 0 || p_sys->hh.h264.i_pps_count == 0)
1067         return VLC_EGENERIC;
1068
1069     unsigned i_h264_width, i_h264_height, i_video_width, i_video_height;
1070     int i_sar_num, i_sar_den, i_ret;
1071     i_ret = h264_helper_get_current_picture_size(&p_sys->hh,
1072                                                  &i_h264_width, &i_h264_height,
1073                                                  &i_video_width, &i_video_height);
1074     if (i_ret != VLC_SUCCESS)
1075         return i_ret;
1076     i_ret = h264_helper_get_current_sar(&p_sys->hh, &i_sar_num, &i_sar_den);
1077     if (i_ret != VLC_SUCCESS)
1078         return i_ret;
1079
1080     p_dec->fmt_out.video.i_visible_width =
1081     p_dec->fmt_out.video.i_width = i_video_width;
1082     p_dec->fmt_out.video.i_visible_height =
1083     p_dec->fmt_out.video.i_height = i_video_height;
1084     p_dec->fmt_out.video.i_sar_num = i_sar_num;
1085     p_dec->fmt_out.video.i_sar_den = i_sar_den;
1086
1087     block_t *p_avcC = h264_helper_get_avcc_config(&p_sys->hh);
1088     if (!p_avcC)
1089         return VLC_EGENERIC;
1090
1091     p_sys->extradataInfo = ExtradataInfoCreate(CFSTR("avcC"),
1092                                                p_avcC->p_buffer, p_avcC->i_buffer);
1093     block_Release(p_avcC);
1094     return (p_sys->extradataInfo == nil) ? VLC_EGENERIC: VLC_SUCCESS;
1095 }
1096
1097 static CFMutableDictionaryRef ExtradataInfoCreate(CFStringRef name,
1098                                                   void *p_data, size_t i_data)
1099 {
1100     CFMutableDictionaryRef extradataInfo = cfdict_create(1);
1101     if (extradataInfo == nil)
1102         return nil;
1103
1104     if (p_data == NULL)
1105         return nil;
1106
1107     CFDataRef extradata = CFDataCreate(kCFAllocatorDefault, p_data, i_data);
1108     if (extradata == nil)
1109     {
1110         CFRelease(extradataInfo);
1111         return nil;
1112     }
1113     CFDictionarySetValue(extradataInfo, name, extradata);
1114     CFRelease(extradata);
1115     return extradataInfo;
1116 }
1117
1118 static CMSampleBufferRef VTSampleBufferCreate(decoder_t *p_dec,
1119                                               CMFormatDescriptionRef fmt_desc,
1120                                               block_t *p_block)
1121 {
1122     OSStatus status;
1123     CMBlockBufferRef  block_buf = NULL;
1124     CMSampleBufferRef sample_buf = NULL;
1125     CMTime pts;
1126     if(!p_dec->p_sys->b_poc_based_reorder && p_block->i_pts == VLC_TS_INVALID)
1127         pts = CMTimeMake(p_block->i_dts, CLOCK_FREQ);
1128     else
1129         pts = CMTimeMake(p_block->i_pts, CLOCK_FREQ);
1130
1131     CMSampleTimingInfo timeInfoArray[1] = { {
1132         .duration = CMTimeMake(p_block->i_length, 1),
1133         .presentationTimeStamp = pts,
1134         .decodeTimeStamp = CMTimeMake(p_block->i_dts, CLOCK_FREQ),
1135     } };
1136
1137     status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault,// structureAllocator
1138                                                 p_block->p_buffer,  // memoryBlock
1139                                                 p_block->i_buffer,  // blockLength
1140                                                 kCFAllocatorNull,   // blockAllocator
1141                                                 NULL,               // customBlockSource
1142                                                 0,                  // offsetToData
1143                                                 p_block->i_buffer,  // dataLength
1144                                                 false,              // flags
1145                                                 &block_buf);
1146
1147     if (!status) {
1148         status = CMSampleBufferCreate(kCFAllocatorDefault,  // allocator
1149                                       block_buf,            // dataBuffer
1150                                       TRUE,                 // dataReady
1151                                       0,                    // makeDataReadyCallback
1152                                       0,                    // makeDataReadyRefcon
1153                                       fmt_desc,             // formatDescription
1154                                       1,                    // numSamples
1155                                       1,                    // numSampleTimingEntries
1156                                       timeInfoArray,        // sampleTimingArray
1157                                       0,                    // numSampleSizeEntries
1158                                       NULL,                 // sampleSizeArray
1159                                       &sample_buf);
1160         if (status != noErr)
1161             msg_Warn(p_dec, "sample buffer creation failure %i", (int)status);
1162     } else
1163         msg_Warn(p_dec, "cm block buffer creation failure %i", (int)status);
1164
1165     if (block_buf != nil)
1166         CFRelease(block_buf);
1167     block_buf = nil;
1168
1169     return sample_buf;
1170 }
1171
1172 static int HandleVTStatus(decoder_t *p_dec, OSStatus status)
1173 {
1174 #define VTERRCASE(x) \
1175     case x: msg_Err(p_dec, "vt session error: '" #x "'"); break;
1176
1177     switch (status)
1178     {
1179         case 0:
1180             return VLC_SUCCESS;
1181
1182         VTERRCASE(kVTPropertyNotSupportedErr)
1183         VTERRCASE(kVTPropertyReadOnlyErr)
1184         VTERRCASE(kVTParameterErr)
1185         VTERRCASE(kVTInvalidSessionErr)
1186         VTERRCASE(kVTAllocationFailedErr)
1187         VTERRCASE(kVTPixelTransferNotSupportedErr)
1188         VTERRCASE(kVTCouldNotFindVideoDecoderErr)
1189         VTERRCASE(kVTCouldNotCreateInstanceErr)
1190         VTERRCASE(kVTCouldNotFindVideoEncoderErr)
1191         VTERRCASE(kVTVideoDecoderBadDataErr)
1192         VTERRCASE(kVTVideoDecoderUnsupportedDataFormatErr)
1193         VTERRCASE(kVTVideoDecoderMalfunctionErr)
1194         VTERRCASE(kVTVideoEncoderMalfunctionErr)
1195         VTERRCASE(kVTVideoDecoderNotAvailableNowErr)
1196         VTERRCASE(kVTImageRotationNotSupportedErr)
1197         VTERRCASE(kVTVideoEncoderNotAvailableNowErr)
1198         VTERRCASE(kVTFormatDescriptionChangeNotSupportedErr)
1199         VTERRCASE(kVTInsufficientSourceColorDataErr)
1200         VTERRCASE(kVTCouldNotCreateColorCorrectionDataErr)
1201         VTERRCASE(kVTColorSyncTransformConvertFailedErr)
1202         VTERRCASE(kVTVideoDecoderAuthorizationErr)
1203         VTERRCASE(kVTVideoEncoderAuthorizationErr)
1204         VTERRCASE(kVTColorCorrectionPixelTransferFailedErr)
1205         VTERRCASE(kVTMultiPassStorageIdentifierMismatchErr)
1206         VTERRCASE(kVTMultiPassStorageInvalidErr)
1207         VTERRCASE(kVTFrameSiloInvalidTimeStampErr)
1208         VTERRCASE(kVTFrameSiloInvalidTimeRangeErr)
1209         VTERRCASE(kVTCouldNotFindTemporalFilterErr)
1210         VTERRCASE(kVTPixelTransferNotPermittedErr)
1211         case -12219:
1212             msg_Err(p_dec, "vt session error: "
1213                     "'kVTColorCorrectionImageRotationFailedErr'");
1214             break;
1215         default:
1216             msg_Err(p_dec, "unknown vt session error (%i)", (int)status);
1217     }
1218 #undef VTERRCASE
1219     return VLC_EGENERIC;
1220 }
1221
1222 #pragma mark - actual decoding
1223
1224 static void Flush(decoder_t *p_dec)
1225 {
1226     decoder_sys_t *p_sys = p_dec->p_sys;
1227
1228     /* There is no Flush in VT api, ask to restart VT from next DecodeBlock if
1229      * we already feed some input blocks (it's better to not restart here in
1230      * order to avoid useless restart just before a close). */
1231     p_sys->b_vt_flush = p_sys->b_vt_feed;
1232 }
1233
1234 static void Drain(decoder_t *p_dec)
1235 {
1236     decoder_sys_t *p_sys = p_dec->p_sys;
1237
1238     /* draining: return last pictures of the reordered queue */
1239     if (p_sys->session)
1240         VTDecompressionSessionWaitForAsynchronousFrames(p_sys->session);
1241
1242     vlc_mutex_lock(&p_sys->lock);
1243     DrainDPB( p_dec );
1244     vlc_mutex_unlock(&p_sys->lock);
1245 }
1246
1247 static int DecodeBlock(decoder_t *p_dec, block_t *p_block)
1248 {
1249     decoder_sys_t *p_sys = p_dec->p_sys;
1250
1251     if (p_sys->b_vt_flush) {
1252         RestartVideoToolbox(p_dec, false);
1253         p_sys->b_vt_flush = false;
1254     }
1255
1256     if (p_block == NULL)
1257     {
1258         Drain(p_dec);
1259         return VLCDEC_SUCCESS;
1260     }
1261
1262     vlc_mutex_lock(&p_sys->lock);
1263     if (p_sys->b_abort) { /* abort from output thread (DecoderCallback) */
1264         vlc_mutex_unlock(&p_sys->lock);
1265         /* Add an empty variable so that videotoolbox won't be loaded again for
1266         * this ES */
1267         var_Create(p_dec, "videotoolbox-failed", VLC_VAR_VOID);
1268         return VLCDEC_RELOAD;
1269     }
1270     vlc_mutex_unlock(&p_sys->lock);
1271
1272     if (unlikely(p_block->i_flags&(BLOCK_FLAG_CORRUPTED)))
1273     {
1274         if (p_sys->b_vt_feed)
1275         {
1276             Drain(p_dec);
1277             RestartVideoToolbox(p_dec, false);
1278         }
1279         goto skip;
1280     }
1281
1282     bool b_config_changed = false;
1283     if (p_sys->codec == kCMVideoCodecType_H264)
1284     {
1285         p_block = p_sys->hh.pf_process_block(&p_sys->hh, p_block, &b_config_changed);
1286         if (!p_block)
1287             return VLCDEC_SUCCESS;
1288     }
1289
1290     frame_info_t *p_info = CreateReorderInfo(p_dec, p_block);
1291     if(unlikely(!p_info))
1292         goto skip;
1293
1294     if (b_config_changed && p_info->b_flush)
1295     {
1296         /* decoding didn't start yet, which is ok for H264, let's see
1297          * if we can use this block to get going */
1298         assert(p_sys->codec == kCMVideoCodecType_H264);
1299         if (p_sys->session)
1300         {
1301             msg_Dbg(p_dec, "SPS/PPS changed: draining H264 decoder");
1302             Drain(p_dec);
1303             msg_Dbg(p_dec, "SPS/PPS changed: restarting H264 decoder");
1304             StopVideoToolbox(p_dec, true);
1305         }
1306
1307         int i_ret = avcCFromAnnexBCreate(p_dec);
1308         if (i_ret == VLC_SUCCESS)
1309         {
1310             msg_Dbg(p_dec, "Got SPS/PPS: late opening of H264 decoder");
1311             StartVideoToolbox(p_dec);
1312         }
1313         if (!p_sys->session)
1314         {
1315             free(p_info);
1316             goto skip;
1317         }
1318     }
1319
1320     CMSampleBufferRef sampleBuffer =
1321         VTSampleBufferCreate(p_dec, p_sys->videoFormatDescription, p_block);
1322     if (unlikely(!sampleBuffer))
1323     {
1324         free(p_info);
1325         goto skip;
1326     }
1327
1328     VTDecodeInfoFlags flagOut;
1329     VTDecodeFrameFlags decoderFlags = kVTDecodeFrame_EnableAsynchronousDecompression;
1330     if (p_sys->b_enable_temporal_processing
1331      && (p_block->i_flags & BLOCK_FLAG_INTERLACED_MASK))
1332         decoderFlags |= kVTDecodeFrame_EnableTemporalProcessing;
1333
1334     OSStatus status =
1335         VTDecompressionSessionDecodeFrame(p_sys->session, sampleBuffer,
1336                                           decoderFlags, p_info, &flagOut);
1337     if (HandleVTStatus(p_dec, status) == VLC_SUCCESS)
1338     {
1339         p_sys->b_vt_feed = true;
1340         if( p_block->i_flags & BLOCK_FLAG_END_OF_SEQUENCE )
1341             Drain( p_dec );
1342     }
1343     else
1344     {
1345         bool b_abort = false;
1346         switch (status)
1347         {
1348             case -8960 /* codecErr */:
1349             case kCVReturnInvalidArgument:
1350             case kVTVideoDecoderMalfunctionErr:
1351                 b_abort = true;
1352                 break;
1353             case -8969 /* codecBadDataErr */:
1354             case kVTVideoDecoderBadDataErr:
1355                 if (RestartVideoToolbox(p_dec, true) == VLC_SUCCESS)
1356                 {
1357                     status = VTDecompressionSessionDecodeFrame(p_sys->session,
1358                                     sampleBuffer, decoderFlags, p_info, &flagOut);
1359
1360                     if (status != 0)
1361                     {
1362                         free( p_info );
1363                         b_abort = true;
1364                     }
1365                 }
1366                 break;
1367             case kVTInvalidSessionErr:
1368                 RestartVideoToolbox(p_dec, true);
1369                 break;
1370         }
1371         if (b_abort)
1372         {
1373             msg_Err(p_dec, "decoder failure, Abort.");
1374             /* The decoder module will be reloaded next time since we already
1375              * modified the input block */
1376             vlc_mutex_lock(&p_sys->lock);
1377             p_dec->p_sys->b_abort = true;
1378             vlc_mutex_unlock(&p_sys->lock);
1379         }
1380     }
1381     CFRelease(sampleBuffer);
1382
1383 skip:
1384     block_Release(p_block);
1385     return VLCDEC_SUCCESS;
1386 }
1387
1388 static int UpdateVideoFormat(decoder_t *p_dec, CVPixelBufferRef imageBuffer)
1389 {
1390     CFDictionaryRef attachments = CVBufferGetAttachments(imageBuffer, kCVAttachmentMode_ShouldPropagate);
1391     NSDictionary *attachmentDict = (NSDictionary *)attachments;
1392 #ifndef NDEBUG
1393     NSLog(@"%@", attachments);
1394 #endif
1395     if (attachmentDict == nil || attachmentDict.count == 0)
1396         return -1;
1397
1398     NSString *colorSpace = attachmentDict[(NSString *)kCVImageBufferYCbCrMatrixKey];
1399     if (colorSpace != nil) {
1400         if ([colorSpace isEqualToString:(NSString *)kCVImageBufferYCbCrMatrix_ITU_R_601_4])
1401             p_dec->fmt_out.video.space = COLOR_SPACE_BT601;
1402         else if ([colorSpace isEqualToString:(NSString *)kCVImageBufferYCbCrMatrix_ITU_R_709_2])
1403             p_dec->fmt_out.video.space = COLOR_SPACE_BT709;
1404         else
1405             p_dec->fmt_out.video.space = COLOR_SPACE_UNDEF;
1406     }
1407
1408     NSString *colorprimary = attachmentDict[(NSString *)kCVImageBufferColorPrimariesKey];
1409     if (colorprimary != nil) {
1410         if ([colorprimary isEqualToString:(NSString *)kCVImageBufferColorPrimaries_SMPTE_C] ||
1411             [colorprimary isEqualToString:(NSString *)kCVImageBufferColorPrimaries_EBU_3213])
1412             p_dec->fmt_out.video.primaries = COLOR_PRIMARIES_BT601_625;
1413         else if ([colorprimary isEqualToString:(NSString *)kCVImageBufferColorPrimaries_ITU_R_709_2])
1414             p_dec->fmt_out.video.primaries = COLOR_PRIMARIES_BT709;
1415         else if ([colorprimary isEqualToString:(NSString *)kCVImageBufferColorPrimaries_P22])
1416             p_dec->fmt_out.video.primaries = COLOR_PRIMARIES_DCI_P3;
1417         else
1418             p_dec->fmt_out.video.primaries = COLOR_PRIMARIES_UNDEF;
1419     }
1420
1421     NSString *transfer = attachmentDict[(NSString *)kCVImageBufferTransferFunctionKey];
1422     if (transfer != nil) {
1423         if ([transfer isEqualToString:(NSString *)kCVImageBufferTransferFunction_ITU_R_709_2] ||
1424             [transfer isEqualToString:(NSString *)kCVImageBufferTransferFunction_SMPTE_240M_1995])
1425             p_dec->fmt_out.video.transfer = TRANSFER_FUNC_BT709;
1426         else
1427             p_dec->fmt_out.video.transfer = TRANSFER_FUNC_UNDEF;
1428     }
1429
1430     NSString *chromaLocation = attachmentDict[(NSString *)kCVImageBufferChromaLocationTopFieldKey];
1431     if (chromaLocation != nil) {
1432         if ([chromaLocation isEqualToString:(NSString *)kCVImageBufferChromaLocation_Left] ||
1433             [chromaLocation isEqualToString:(NSString *)kCVImageBufferChromaLocation_DV420])
1434             p_dec->fmt_out.video.chroma_location = CHROMA_LOCATION_LEFT;
1435         else if ([chromaLocation isEqualToString:(NSString *)kCVImageBufferChromaLocation_Center])
1436             p_dec->fmt_out.video.chroma_location = CHROMA_LOCATION_CENTER;
1437         else if ([chromaLocation isEqualToString:(NSString *)kCVImageBufferChromaLocation_TopLeft])
1438             p_dec->fmt_out.video.chroma_location = CHROMA_LOCATION_TOP_LEFT;
1439         else if ([chromaLocation isEqualToString:(NSString *)kCVImageBufferChromaLocation_Top])
1440             p_dec->fmt_out.video.chroma_location = CHROMA_LOCATION_TOP_CENTER;
1441         else
1442             p_dec->fmt_out.video.chroma_location = CHROMA_LOCATION_UNDEF;
1443     }
1444     if (p_dec->fmt_out.video.chroma_location == CHROMA_LOCATION_UNDEF) {
1445         chromaLocation = attachmentDict[(NSString *)kCVImageBufferChromaLocationBottomFieldKey];
1446         if (chromaLocation != nil) {
1447             if ([chromaLocation isEqualToString:(NSString *)kCVImageBufferChromaLocation_BottomLeft])
1448                 p_dec->fmt_out.video.chroma_location = CHROMA_LOCATION_BOTTOM_LEFT;
1449             else if ([chromaLocation isEqualToString:(NSString *)kCVImageBufferChromaLocation_Bottom])
1450                 p_dec->fmt_out.video.chroma_location = CHROMA_LOCATION_BOTTOM_CENTER;
1451         }
1452     }
1453
1454     uint32_t cvfmt = CVPixelBufferGetPixelFormatType(imageBuffer);
1455     msg_Info(p_dec, "vt cvpx chroma: %4.4s",
1456              (const char *)&(uint32_t) { htonl(cvfmt) });
1457     switch (cvfmt)
1458     {
1459         case kCVPixelFormatType_422YpCbCr8:
1460         case 'yuv2':
1461             p_dec->fmt_out.i_codec = VLC_CODEC_CVPX_UYVY;
1462             assert(CVPixelBufferIsPlanar(imageBuffer) == false);
1463             break;
1464         case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
1465         case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
1466             p_dec->fmt_out.i_codec = VLC_CODEC_CVPX_NV12;
1467             assert(CVPixelBufferIsPlanar(imageBuffer) == true);
1468             break;
1469         case kCVPixelFormatType_420YpCbCr8Planar:
1470             p_dec->fmt_out.i_codec = VLC_CODEC_CVPX_I420;
1471             assert(CVPixelBufferIsPlanar(imageBuffer) == true);
1472             break;
1473         case kCVPixelFormatType_32BGRA:
1474             p_dec->fmt_out.i_codec = VLC_CODEC_CVPX_BGRA;
1475             assert(CVPixelBufferIsPlanar(imageBuffer) == false);
1476             break;
1477         default:
1478             p_dec->p_sys->b_abort = true;
1479             return -1;
1480     }
1481     return decoder_UpdateVideoFormat(p_dec);
1482 }
1483
1484 static void DecoderCallback(void *decompressionOutputRefCon,
1485                             void *sourceFrameRefCon,
1486                             OSStatus status,
1487                             VTDecodeInfoFlags infoFlags,
1488                             CVPixelBufferRef imageBuffer,
1489                             CMTime pts,
1490                             CMTime duration)
1491 {
1492     VLC_UNUSED(duration);
1493     decoder_t *p_dec = (decoder_t *)decompressionOutputRefCon;
1494     decoder_sys_t *p_sys = p_dec->p_sys;
1495     frame_info_t *p_info = (frame_info_t *) sourceFrameRefCon;
1496
1497     if (status != noErr) {
1498         msg_Warn(p_dec, "decoding of a frame failed (%i, %u)", (int)status, (unsigned int) infoFlags);
1499         if( status != kVTVideoDecoderBadDataErr && status != -8969 )
1500             free(p_info);
1501         return;
1502     }
1503     assert(imageBuffer);
1504
1505     if (unlikely(!p_sys->b_format_propagated)) {
1506         vlc_mutex_lock(&p_sys->lock);
1507         p_sys->b_format_propagated =
1508             UpdateVideoFormat(p_dec, imageBuffer) == VLC_SUCCESS;
1509         vlc_mutex_unlock(&p_sys->lock);
1510
1511         if (!p_sys->b_format_propagated)
1512         {
1513             free(p_info);
1514             return;
1515         }
1516         assert(p_dec->fmt_out.i_codec != 0);
1517     }
1518
1519     if (infoFlags & kVTDecodeInfo_FrameDropped)
1520     {
1521         /* We can't trust VT, some decoded frames can be marked as dropped */
1522         msg_Dbg(p_dec, "decoder dropped frame");
1523     }
1524
1525     if (!CMTIME_IS_VALID(pts))
1526     {
1527         free(p_info);
1528         return;
1529     }
1530
1531     if (CVPixelBufferGetDataSize(imageBuffer) == 0)
1532     {
1533         free(p_info);
1534         return;
1535     }
1536
1537     if(likely(p_info))
1538     {
1539         picture_t *p_pic = decoder_NewPicture(p_dec);
1540         if (!p_pic)
1541         {
1542             free(p_info);
1543             return;
1544         }
1545
1546         if (cvpxpic_attach(p_pic, imageBuffer) != VLC_SUCCESS)
1547         {
1548             free(p_info);
1549             return;
1550         }
1551
1552         p_info->p_picture = p_pic;
1553
1554         p_pic->date = pts.value;
1555         p_pic->b_force = p_info->b_forced;
1556         p_pic->b_progressive = p_sys->b_handle_deint || p_info->b_progressive;
1557         if(!p_pic->b_progressive)
1558         {
1559             p_pic->i_nb_fields = p_info->i_num_ts;
1560             p_pic->b_top_field_first = p_info->b_top_field_first;
1561         }
1562
1563         vlc_mutex_lock(&p_sys->lock);
1564         OnDecodedFrame( p_dec, p_info );
1565         vlc_mutex_unlock(&p_sys->lock);
1566     }
1567
1568     return;
1569 }