modules/codec/videotoolbox.m

   1 /*****************************************************************************
   2  * videotoolbox.m: Video Toolbox decoder
   3  *****************************************************************************
   4  * Copyright © 2014-2015 VideoLabs SAS
   5  *
   6  * Authors: Felix Paul Kühne <fkuehne # videolan.org>
   7  *
   8  * This program is free software; you can redistribute it and/or modify it
   9  * under the terms of the GNU Lesser General Public License as published by
  10  * the Free Software Foundation; either version 2.1 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16  * GNU Lesser General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU Lesser General Public License
  19  * along with this program; if not, write to the Free Software Foundation,
  20  * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  21  *****************************************************************************/
  22
  23 #pragma mark preamble
  24
  25 #ifdef HAVE_CONFIG_H
  26 # import "config.h"
  27 #endif
  28
  29 #import <vlc_common.h>
  30 #import <vlc_plugin.h>
  31 #import <vlc_codec.h>
  32 #import "hxxx_helper.h"
  33 #import <vlc_bits.h>
  34 #import <vlc_boxes.h>
  35 #import "../packetizer/h264_nal.h"
  36 #import "../packetizer/h264_slice.h"
  37 #import "../packetizer/hxxx_nal.h"
  38 #import "../packetizer/hxxx_sei.h"
  39 #import "../video_chroma/copy.h"
  40
  41 #import <VideoToolbox/VideoToolbox.h>
  42 #import <VideoToolbox/VTErrors.h>
  43
  44 #import <Foundation/Foundation.h>
  45 #import <TargetConditionals.h>
  46
  47 #import <sys/types.h>
  48 #import <sys/sysctl.h>
  49 #import <mach/machine.h>
  50
  51 #if TARGET_OS_IPHONE
  52 #import <UIKit/UIKit.h>
  53
  54 /* support iOS SDKs < v9.1 */
  55 #ifndef CPUFAMILY_ARM_TWISTER
  56 #define CPUFAMILY_ARM_TWISTER 0x92fb37c8
  57 #endif
  58
  59 #endif
  60
  61 #pragma mark - module descriptor
  62
  63 static int OpenDecoder(vlc_object_t *);
  64 static void CloseDecoder(vlc_object_t *);
  65
  66 #if MAC_OS_X_VERSION_MIN_ALLOWED < 1090
  67 const CFStringRef kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder = CFSTR("EnableHardwareAcceleratedVideoDecoder");
  68 const CFStringRef kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder = CFSTR("RequireHardwareAcceleratedVideoDecoder");
  69 #endif
  70
  71 #define VT_REQUIRE_HW_DEC N_("Use Hardware decoders only")
  72 #define VT_TEMPO_DEINTERLACE N_("Deinterlacing")
  73 #define VT_TEMPO_DEINTERLACE_LONG N_("If interlaced content is detected, temporal deinterlacing is enabled at the expense of a pipeline delay.")
  74
  75 vlc_module_begin()
  76 set_category(CAT_INPUT)
  77 set_subcategory(SUBCAT_INPUT_VCODEC)
  78 set_description(N_("VideoToolbox video decoder"))
  79 set_capability("decoder",800)
  80 set_callbacks(OpenDecoder, CloseDecoder)
  81
  82 add_bool("videotoolbox-temporal-deinterlacing", true, VT_TEMPO_DEINTERLACE, VT_TEMPO_DEINTERLACE_LONG, false)
  83 add_bool("videotoolbox-hw-decoder-only", false, VT_REQUIRE_HW_DEC, VT_REQUIRE_HW_DEC, false)
  84 vlc_module_end()
  85
  86 #pragma mark - local prototypes
  87
  88 static int ESDSCreate(decoder_t *, uint8_t *, uint32_t);
  89 static int avcCFromAnnexBCreate(decoder_t *);
  90 static int ExtradataInfoCreate(decoder_t *, CFStringRef, void *, size_t);
  91 static int HandleVTStatus(decoder_t *, OSStatus);
  92 static int DecodeBlock(decoder_t *, block_t *);
  93 static void Flush(decoder_t *);
  94 static void DecoderCallback(void *, void *, OSStatus, VTDecodeInfoFlags,
  95                             CVPixelBufferRef, CMTime, CMTime);
  96 void VTDictionarySetInt32(CFMutableDictionaryRef, CFStringRef, int);
  97 static void copy420YpCbCr8Planar(picture_t *, CVPixelBufferRef buffer,
  98                                  unsigned i_width, unsigned i_height);
  99 static BOOL deviceSupportsAdvancedProfiles();
 100 static BOOL deviceSupportsAdvancedLevels();
 101
 102 struct picture_sys_t {
 103     CFTypeRef pixelBuffer;
 104 };
 105
 106 typedef struct frame_info_t frame_info_t;
 107
 108 struct frame_info_t
 109 {
 110     picture_t *p_picture;
 111     int i_poc;
 112     int i_foc;
 113     bool b_flush;
 114     bool b_field;
 115     bool b_progressive;
 116     uint8_t i_num_ts;
 117     unsigned i_length;
 118     frame_info_t *p_next;
 119 };
 120
 121 #pragma mark - decoder structure
 122
 123 #define H264_MAX_DPB 16
 124
 125 struct decoder_sys_t
 126 {
 127     CMVideoCodecType            codec;
 128     struct                      hxxx_helper hh;
 129
 130     bool                        b_vt_feed;
 131     bool                        b_vt_flush;
 132     VTDecompressionSessionRef   session;
 133     CMVideoFormatDescriptionRef videoFormatDescription;
 134     CFMutableDictionaryRef      decoderConfiguration;
 135     CFMutableDictionaryRef      destinationPixelBufferAttributes;
 136     CFMutableDictionaryRef      extradataInfo;
 137
 138     vlc_mutex_t                 lock;
 139     frame_info_t               *p_pic_reorder;
 140     uint8_t                     i_pic_reorder;
 141     uint8_t                     i_pic_reorder_max;
 142     bool                        b_poc_based_reorder;
 143     bool                        b_enable_temporal_processing;
 144
 145     bool                        b_format_propagated;
 146     bool                        b_abort;
 147
 148     poc_context_t               pocctx;
 149     date_t                      pts;
 150 };
 151
 152 #pragma mark - start & stop
 153
 154 static void GetSPSPPS(uint8_t i_pps_id, void *priv,
 155                       const h264_sequence_parameter_set_t **pp_sps,
 156                       const h264_picture_parameter_set_t **pp_pps)
 157 {
 158     decoder_sys_t *p_sys = priv;
 159
 160     *pp_pps = p_sys->hh.h264.pps_list[i_pps_id].h264_pps;
 161     if(*pp_pps == NULL)
 162         *pp_sps = NULL;
 163     else
 164         *pp_sps = p_sys->hh.h264.sps_list[(*pp_pps)->i_sps_id].h264_sps;
 165 }
 166
 167 struct sei_callback_s
 168 {
 169     uint8_t i_pic_struct;
 170     const h264_sequence_parameter_set_t *p_sps;
 171 };
 172
 173 static bool ParseH264SEI(const hxxx_sei_data_t *p_sei_data, void *priv)
 174 {
 175
 176     if(p_sei_data->i_type == HXXX_SEI_PIC_TIMING)
 177     {
 178         struct sei_callback_s *s = priv;
 179         if(s->p_sps && s->p_sps->vui.b_valid)
 180         {
 181             if(s->p_sps->vui.b_hrd_parameters_present_flag)
 182             {
 183                 bs_read(p_sei_data->p_bs, s->p_sps->vui.i_cpb_removal_delay_length_minus1 + 1);
 184                 bs_read(p_sei_data->p_bs, s->p_sps->vui.i_dpb_output_delay_length_minus1 + 1);
 185             }
 186
 187             if(s->p_sps->vui.b_pic_struct_present_flag)
 188                s->i_pic_struct = bs_read( p_sei_data->p_bs, 4);
 189         }
 190         return false;
 191     }
 192
 193     return true;
 194 }
 195
 196 static bool ParseH264NAL(decoder_sys_t *p_sys,
 197                          const uint8_t *p_buffer, size_t i_buffer,
 198                          uint8_t i_nal_length_size, frame_info_t *p_info)
 199 {
 200     hxxx_iterator_ctx_t itctx;
 201     hxxx_iterator_init(&itctx, p_buffer, i_buffer, i_nal_length_size);
 202
 203     const uint8_t *p_nal; size_t i_nal;
 204     const uint8_t *p_sei_nal = NULL; size_t i_sei_nal = 0;
 205     while(hxxx_iterate_next(&itctx, &p_nal, &i_nal))
 206     {
 207         if(i_nal < 2)
 208             continue;
 209
 210         const enum h264_nal_unit_type_e i_nal_type = p_nal[0] & 0x1F;
 211
 212         if (i_nal_type <= H264_NAL_SLICE_IDR && i_nal_type != H264_NAL_UNKNOWN)
 213         {
 214             h264_slice_t slice;
 215             if(!h264_decode_slice(p_nal, i_nal, GetSPSPPS, p_sys, &slice))
 216                 return false;
 217
 218             const h264_sequence_parameter_set_t *p_sps;
 219             const h264_picture_parameter_set_t *p_pps;
 220             GetSPSPPS(slice.i_pic_parameter_set_id, p_sys, &p_sps, &p_pps);
 221             if(p_sps)
 222             {
 223                 unsigned dummy;
 224                 uint8_t i_reorder;
 225                 h264_get_dpb_values(p_sps, &i_reorder, &dummy);
 226                 vlc_mutex_lock(&p_sys->lock);
 227                 p_sys->i_pic_reorder_max = i_reorder;
 228                 vlc_mutex_unlock(&p_sys->lock);
 229
 230                 int bFOC;
 231                 h264_compute_poc(p_sps, &slice, &p_sys->pocctx,
 232                                  &p_info->i_poc, &p_info->i_foc, &bFOC);
 233
 234                 p_info->b_flush = (slice.type == H264_SLICE_TYPE_I) || slice.has_mmco5;
 235                 p_info->b_field = slice.i_field_pic_flag;
 236                 p_info->b_progressive = !p_sps->mb_adaptive_frame_field_flag &&
 237                                         !slice.i_field_pic_flag;
 238
 239                 struct sei_callback_s sei;
 240                 sei.p_sps = p_sps;
 241                 sei.i_pic_struct = UINT8_MAX;
 242
 243                 if(p_sei_nal)
 244                     HxxxParseSEI(p_sei_nal, i_sei_nal, 1, ParseH264SEI, &sei);
 245
 246                 p_info->i_num_ts = h264_get_num_ts(p_sps, &slice, sei.i_pic_struct,
 247                                                    p_info->i_foc, bFOC);
 248             }
 249
 250             return true; /* No need to parse further NAL */
 251         }
 252         else if(i_nal_type == H264_NAL_SEI)
 253         {
 254             p_sei_nal = p_nal;
 255             i_sei_nal = i_nal;
 256         }
 257     }
 258
 259     return false;
 260 }
 261
 262 static void InsertIntoDPB(decoder_sys_t *p_sys, frame_info_t *p_info)
 263 {
 264     frame_info_t **pp_lead_in = &p_sys->p_pic_reorder;
 265
 266     for( ;; pp_lead_in = & ((*pp_lead_in)->p_next))
 267     {
 268         bool b_insert;
 269         if(*pp_lead_in == NULL)
 270             b_insert = true;
 271         else if(p_sys->b_poc_based_reorder)
 272             b_insert = ((*pp_lead_in)->i_foc > p_info->i_foc);
 273         else
 274             b_insert = ((*pp_lead_in)->p_picture->date >= p_info->p_picture->date);
 275
 276         if(b_insert)
 277         {
 278             p_info->p_next = *pp_lead_in;
 279             *pp_lead_in = p_info;
 280             p_sys->i_pic_reorder += (p_info->b_field) ? 1 : 2;
 281             break;
 282         }
 283     }
 284 #if 0
 285     for(frame_info_t *p_in=p_sys->p_pic_reorder; p_in; p_in = p_in->p_next)
 286         printf(" %d", p_in->i_foc);
 287     printf("\n");
 288 #endif
 289 }
 290
 291 static picture_t * RemoveOneFrameFromDPB(decoder_sys_t *p_sys)
 292 {
 293     frame_info_t *p_info = p_sys->p_pic_reorder;
 294     if(p_info == NULL)
 295         return NULL;
 296
 297     const int i_framepoc = p_info->i_poc;
 298
 299     picture_t *p_ret = NULL;
 300     picture_t **pp_ret_last = &p_ret;
 301     bool b_dequeue;
 302
 303     do
 304     {
 305         picture_t *p_field = p_info->p_picture;
 306
 307         /* Compute time if missing */
 308         if (p_field->date == VLC_TS_INVALID)
 309             p_field->date = date_Get(&p_sys->pts);
 310         else
 311             date_Set(&p_sys->pts, p_field->date);
 312
 313         /* Set next picture time, in case it is missing */
 314         if (p_info->i_length)
 315             date_Set(&p_sys->pts, p_field->date + p_info->i_length);
 316         else
 317             date_Increment(&p_sys->pts, p_info->i_num_ts);
 318
 319         *pp_ret_last = p_field;
 320         pp_ret_last = &p_field->p_next;
 321
 322         p_sys->i_pic_reorder -= (p_info->b_field) ? 1 : 2;
 323
 324         p_sys->p_pic_reorder = p_info->p_next;
 325         free(p_info);
 326         p_info = p_sys->p_pic_reorder;
 327
 328         if (p_info)
 329         {
 330             if (p_sys->b_poc_based_reorder)
 331                 b_dequeue = (p_info->i_poc == i_framepoc);
 332             else
 333                 b_dequeue = (p_field->date == p_info->p_picture->date);
 334         }
 335         else b_dequeue = false;
 336
 337     } while(b_dequeue);
 338
 339     return p_ret;
 340 }
 341
 342 static void FlushDPB(decoder_t *p_dec)
 343 {
 344     decoder_sys_t *p_sys = p_dec->p_sys;
 345     for( ;; )
 346     {
 347         picture_t *p_fields = RemoveOneFrameFromDPB(p_sys);
 348         if (p_fields == NULL)
 349             break;
 350         do
 351         {
 352             picture_t *p_next = p_fields->p_next;
 353             p_fields->p_next = NULL;
 354             decoder_QueueVideo(p_dec, p_fields);
 355             p_fields = p_next;
 356         } while(p_fields != NULL);
 357     }
 358 }
 359
 360 static frame_info_t * CreateReorderInfo(decoder_sys_t *p_sys, const block_t *p_block)
 361 {
 362     frame_info_t *p_info = calloc(1, sizeof(*p_info));
 363     if (!p_info)
 364         return NULL;
 365
 366     if (p_sys->b_poc_based_reorder)
 367     {
 368         if(p_sys->codec != kCMVideoCodecType_H264 ||
 369            !ParseH264NAL(p_sys, p_block->p_buffer, p_block->i_buffer, 4, p_info))
 370         {
 371             assert(p_sys->codec == kCMVideoCodecType_H264);
 372             free(p_info);
 373             return NULL;
 374         }
 375     }
 376     else
 377     {
 378         p_info->i_num_ts = 2;
 379         p_info->b_progressive = true;
 380         p_info->b_field = false;
 381     }
 382
 383     p_info->i_length = p_block->i_length;
 384
 385     if (date_Get(&p_sys->pts) == VLC_TS_INVALID)
 386         date_Set(&p_sys->pts, p_block->i_dts);
 387
 388     return p_info;
 389 }
 390
 391 static void OnDecodedFrame(decoder_t *p_dec, frame_info_t *p_info)
 392 {
 393     decoder_sys_t *p_sys = p_dec->p_sys;
 394     assert(p_info->p_picture);
 395     while(p_info->b_flush || p_sys->i_pic_reorder == (p_sys->i_pic_reorder_max * 2))
 396     {
 397         picture_t *p_fields = RemoveOneFrameFromDPB(p_sys);
 398         if (p_fields == NULL)
 399             break;
 400         do
 401         {
 402             picture_t *p_next = p_fields->p_next;
 403             p_fields->p_next = NULL;
 404             decoder_QueueVideo(p_dec, p_fields);
 405             p_fields = p_next;
 406         } while(p_fields != NULL);
 407     }
 408
 409     InsertIntoDPB(p_sys, p_info);
 410 }
 411
 412 static CMVideoCodecType CodecPrecheck(decoder_t *p_dec)
 413 {
 414     uint8_t i_profile = 0xFF, i_level = 0xFF;
 415     bool b_ret = false;
 416     CMVideoCodecType codec;
 417
 418     /* check for the codec we can and want to decode */
 419     switch (p_dec->fmt_in.i_codec) {
 420         case VLC_CODEC_H264:
 421             codec = kCMVideoCodecType_H264;
 422
 423             b_ret = h264_get_profile_level(&p_dec->fmt_in, &i_profile, &i_level, NULL);
 424             if (!b_ret) {
 425                 msg_Warn(p_dec, "H264 profile and level parsing failed because it didn't arrive yet");
 426                 return kCMVideoCodecType_H264;
 427             }
 428
 429             msg_Dbg(p_dec, "trying to decode MPEG-4 Part 10: profile %" PRIx8 ", level %" PRIx8, i_profile, i_level);
 430
 431             switch (i_profile) {
 432                 case PROFILE_H264_BASELINE:
 433                 case PROFILE_H264_MAIN:
 434                 case PROFILE_H264_HIGH:
 435                     break;
 436
 437                 case PROFILE_H264_HIGH_10:
 438                 {
 439                     if (deviceSupportsAdvancedProfiles())
 440                         break;
 441                 }
 442
 443                 default:
 444                 {
 445                     msg_Dbg(p_dec, "unsupported H264 profile %" PRIx8, i_profile);
 446                     return -1;
 447                 }
 448             }
 449
 450 #if !TARGET_OS_IPHONE
 451             /* a level higher than 5.2 was not tested, so don't dare to
 452              * try to decode it*/
 453             if (i_level > 52) {
 454                 msg_Dbg(p_dec, "unsupported H264 level %" PRIx8, i_level);
 455                 return -1;
 456             }
 457 #else
 458             /* on SoC A8, 4.2 is the highest specified profile */
 459             if (i_level > 42) {
 460                 /* on Twister, we can do up to 5.2 */
 461                 if (!deviceSupportsAdvancedLevels() || i_level > 52) {
 462                     msg_Dbg(p_dec, "unsupported H264 level %" PRIx8, i_level);
 463                     return -1;
 464                 }
 465             }
 466 #endif
 467
 468             break;
 469         case VLC_CODEC_MP4V:
 470         {
 471             if (p_dec->fmt_in.i_original_fourcc == VLC_FOURCC( 'X','V','I','D' )) {
 472                 msg_Warn(p_dec, "XVID decoding not implemented, fallback on software");
 473                 return -1;
 474             }
 475
 476             msg_Dbg(p_dec, "Will decode MP4V with original FourCC '%4.4s'", (char *)&p_dec->fmt_in.i_original_fourcc);
 477             codec = kCMVideoCodecType_MPEG4Video;
 478
 479             break;
 480         }
 481 #if !TARGET_OS_IPHONE
 482         case VLC_CODEC_H263:
 483             codec = kCMVideoCodecType_H263;
 484             break;
 485
 486             /* there are no DV or ProRes decoders on iOS, so bailout early */
 487         case VLC_CODEC_PRORES:
 488             /* the VT decoder can't differenciate between the ProRes flavors, so we do it */
 489             switch (p_dec->fmt_in.i_original_fourcc) {
 490                 case VLC_FOURCC( 'a','p','4','c' ):
 491                 case VLC_FOURCC( 'a','p','4','h' ):
 492                     codec = kCMVideoCodecType_AppleProRes4444;
 493                     break;
 494
 495                 case VLC_FOURCC( 'a','p','c','h' ):
 496                     codec = kCMVideoCodecType_AppleProRes422HQ;
 497                     break;
 498
 499                 case VLC_FOURCC( 'a','p','c','s' ):
 500                     codec = kCMVideoCodecType_AppleProRes422LT;
 501                     break;
 502
 503                 case VLC_FOURCC( 'a','p','c','o' ):
 504                     codec = kCMVideoCodecType_AppleProRes422Proxy;
 505                     break;
 506
 507                 default:
 508                     codec = kCMVideoCodecType_AppleProRes422;
 509                     break;
 510             }
 511             if (codec != 0)
 512                 break;
 513
 514         case VLC_CODEC_DV:
 515             /* the VT decoder can't differenciate between PAL and NTSC, so we need to do it */
 516             switch (p_dec->fmt_in.i_original_fourcc) {
 517                 case VLC_FOURCC( 'd', 'v', 'c', ' '):
 518                 case VLC_FOURCC( 'd', 'v', ' ', ' '):
 519                     msg_Dbg(p_dec, "Decoding DV NTSC");
 520                     codec = kCMVideoCodecType_DVCNTSC;
 521                     break;
 522
 523                 case VLC_FOURCC( 'd', 'v', 's', 'd'):
 524                 case VLC_FOURCC( 'd', 'v', 'c', 'p'):
 525                 case VLC_FOURCC( 'D', 'V', 'S', 'D'):
 526                     msg_Dbg(p_dec, "Decoding DV PAL");
 527                     codec = kCMVideoCodecType_DVCPAL;
 528                     break;
 529
 530                 default:
 531                     break;
 532             }
 533             if (codec != 0)
 534                 break;
 535 #endif
 536             /* mpgv / mp2v needs fixing, so disable it for now */
 537 #if 0
 538         case VLC_CODEC_MPGV:
 539             codec = kCMVideoCodecType_MPEG1Video;
 540             break;
 541         case VLC_CODEC_MP2V:
 542             codec = kCMVideoCodecType_MPEG2Video;
 543             break;
 544 #endif
 545
 546         default:
 547 #ifndef NDEBUG
 548             msg_Err(p_dec, "'%4.4s' is not supported", (char *)&p_dec->fmt_in.i_codec);
 549 #endif
 550             return -1;
 551     }
 552
 553     return codec;
 554 }
 555
 556 static int StartVideoToolbox(decoder_t *p_dec)
 557 {
 558     decoder_sys_t *p_sys = p_dec->p_sys;
 559     OSStatus status;
 560
 561     assert(p_sys->extradataInfo != nil);
 562
 563     p_sys->decoderConfiguration =
 564         CFDictionaryCreateMutable(kCFAllocatorDefault, 2,
 565                                   &kCFTypeDictionaryKeyCallBacks,
 566                                   &kCFTypeDictionaryValueCallBacks);
 567     if (p_sys->decoderConfiguration == NULL)
 568         return VLC_ENOMEM;
 569
 570     CFDictionarySetValue(p_sys->decoderConfiguration,
 571                          kCVImageBufferChromaLocationBottomFieldKey,
 572                          kCVImageBufferChromaLocation_Left);
 573     CFDictionarySetValue(p_sys->decoderConfiguration,
 574                          kCVImageBufferChromaLocationTopFieldKey,
 575                          kCVImageBufferChromaLocation_Left);
 576
 577     CFDictionarySetValue(p_sys->decoderConfiguration,
 578                          kCMFormatDescriptionExtension_SampleDescriptionExtensionAtoms,
 579                          p_sys->extradataInfo);
 580
 581     /* pixel aspect ratio */
 582     CFMutableDictionaryRef pixelaspectratio =
 583         CFDictionaryCreateMutable(kCFAllocatorDefault, 2,
 584                                   &kCFTypeDictionaryKeyCallBacks,
 585                                   &kCFTypeDictionaryValueCallBacks);
 586
 587     const unsigned i_video_width = p_dec->fmt_out.video.i_width;
 588     const unsigned i_video_height = p_dec->fmt_out.video.i_height;
 589     const unsigned i_sar_num = p_dec->fmt_out.video.i_sar_num;
 590     const unsigned i_sar_den = p_dec->fmt_out.video.i_sar_den;
 591
 592
 593     date_Init( &p_sys->pts, p_dec->fmt_in.video.i_frame_rate * 2, p_dec->fmt_in.video.i_frame_rate_base );
 594
 595     VTDictionarySetInt32(pixelaspectratio,
 596                          kCVImageBufferPixelAspectRatioHorizontalSpacingKey,
 597                          i_sar_num);
 598     VTDictionarySetInt32(pixelaspectratio,
 599                          kCVImageBufferPixelAspectRatioVerticalSpacingKey,
 600                          i_sar_den);
 601     CFDictionarySetValue(p_sys->decoderConfiguration,
 602                          kCVImageBufferPixelAspectRatioKey,
 603                          pixelaspectratio);
 604     CFRelease(pixelaspectratio);
 605
 606     /* enable HW accelerated playback, since this is optional on OS X
 607      * note that the backend may still fallback on software mode if no
 608      * suitable hardware is available */
 609     CFDictionarySetValue(p_sys->decoderConfiguration,
 610                          kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder,
 611                          kCFBooleanTrue);
 612
 613     /* on OS X, we can force VT to fail if no suitable HW decoder is available,
 614      * preventing the aforementioned SW fallback */
 615     if (var_InheritInteger(p_dec, "videotoolbox-hw-decoder-only"))
 616         CFDictionarySetValue(p_sys->decoderConfiguration,
 617                              kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder,
 618                              kCFBooleanTrue);
 619
 620     if (p_sys->b_enable_temporal_processing)
 621     {
 622         msg_Dbg(p_dec, "Interlaced content detected, inserting temporal deinterlacer");
 623         CFDictionarySetValue(p_sys->decoderConfiguration,
 624                              kVTDecompressionPropertyKey_FieldMode,
 625                              kVTDecompressionProperty_FieldMode_DeinterlaceFields);
 626         CFDictionarySetValue(p_sys->decoderConfiguration,
 627                              kVTDecompressionPropertyKey_DeinterlaceMode,
 628                              kVTDecompressionProperty_DeinterlaceMode_Temporal);
 629     }
 630
 631     /* create video format description */
 632     status = CMVideoFormatDescriptionCreate(kCFAllocatorDefault,
 633                                             p_sys->codec,
 634                                             i_video_width,
 635                                             i_video_height,
 636                                             p_sys->decoderConfiguration,
 637                                             &p_sys->videoFormatDescription);
 638     if (status) {
 639         CFRelease(p_sys->decoderConfiguration);
 640         msg_Err(p_dec, "video format description creation failed (%i)", (int)status);
 641         return VLC_EGENERIC;
 642     }
 643
 644     /* destination pixel buffer attributes */
 645     p_sys->destinationPixelBufferAttributes = CFDictionaryCreateMutable(kCFAllocatorDefault,
 646                                                                         2,
 647                                                                         &kCFTypeDictionaryKeyCallBacks,
 648                                                                         &kCFTypeDictionaryValueCallBacks);
 649
 650 #if !TARGET_OS_IPHONE
 651     CFDictionarySetValue(p_sys->destinationPixelBufferAttributes,
 652                          kCVPixelBufferIOSurfaceOpenGLTextureCompatibilityKey,
 653                          kCFBooleanTrue);
 654 #else
 655     CFDictionarySetValue(p_sys->destinationPixelBufferAttributes,
 656                          kCVPixelBufferOpenGLESCompatibilityKey,
 657                          kCFBooleanTrue);
 658 #endif
 659
 660     VTDictionarySetInt32(p_sys->destinationPixelBufferAttributes,
 661                          kCVPixelBufferWidthKey,
 662                          i_video_width);
 663     VTDictionarySetInt32(p_sys->destinationPixelBufferAttributes,
 664                          kCVPixelBufferHeightKey,
 665                          i_video_height);
 666     VTDictionarySetInt32(p_sys->destinationPixelBufferAttributes,
 667                          kCVPixelBufferBytesPerRowAlignmentKey,
 668                          i_video_width * 2);
 669
 670     /* setup decoder callback record */
 671     VTDecompressionOutputCallbackRecord decoderCallbackRecord;
 672     decoderCallbackRecord.decompressionOutputCallback = DecoderCallback;
 673     decoderCallbackRecord.decompressionOutputRefCon = p_dec;
 674
 675     /* create decompression session */
 676     status = VTDecompressionSessionCreate(kCFAllocatorDefault,
 677                                           p_sys->videoFormatDescription,
 678                                           p_sys->decoderConfiguration,
 679                                           p_sys->destinationPixelBufferAttributes,
 680                                           &decoderCallbackRecord, &p_sys->session);
 681
 682     if (HandleVTStatus(p_dec, status) != VLC_SUCCESS)
 683         return VLC_EGENERIC;
 684
 685     return VLC_SUCCESS;
 686 }
 687
 688 static void StopVideoToolbox(decoder_t *p_dec, bool b_reset_format)
 689 {
 690     decoder_sys_t *p_sys = p_dec->p_sys;
 691
 692     if (p_sys->session != nil)
 693     {
 694         VTDecompressionSessionInvalidate(p_sys->session);
 695         CFRelease(p_sys->session);
 696         p_sys->session = nil;
 697
 698         if (b_reset_format)
 699         {
 700             p_sys->b_format_propagated = false;
 701             p_dec->fmt_out.i_codec = 0;
 702         }
 703         FlushDPB( p_dec );
 704     }
 705
 706     if (p_sys->videoFormatDescription != nil) {
 707         CFRelease(p_sys->videoFormatDescription);
 708         p_sys->videoFormatDescription = nil;
 709     }
 710     if (p_sys->decoderConfiguration != nil) {
 711         CFRelease(p_sys->decoderConfiguration);
 712         p_sys->decoderConfiguration = nil;
 713     }
 714     if (p_sys->destinationPixelBufferAttributes != nil) {
 715         CFRelease(p_sys->destinationPixelBufferAttributes);
 716         p_sys->destinationPixelBufferAttributes = nil;
 717     }
 718 }
 719
 720 static int RestartVideoToolbox(decoder_t *p_dec, bool b_reset_format)
 721 {
 722     decoder_sys_t *p_sys = p_dec->p_sys;
 723
 724     msg_Dbg(p_dec, "Restarting decoder session");
 725
 726     if (p_sys->session != nil)
 727         StopVideoToolbox(p_dec, b_reset_format);
 728
 729     return StartVideoToolbox(p_dec);
 730 }
 731
 732 #pragma mark - module open and close
 733
 734 static int SetupDecoderExtradata(decoder_t *p_dec)
 735 {
 736     decoder_sys_t *p_sys = p_dec->p_sys;
 737     CFMutableDictionaryRef extradata_info = NULL;
 738
 739     if (p_sys->codec == kCMVideoCodecType_H264)
 740     {
 741         hxxx_helper_init(&p_sys->hh, VLC_OBJECT(p_dec),
 742                          p_dec->fmt_in.i_codec, true);
 743         int i_ret = hxxx_helper_set_extra(&p_sys->hh, p_dec->fmt_in.p_extra,
 744                                           p_dec->fmt_in.i_extra);
 745         if (i_ret != VLC_SUCCESS)
 746             return i_ret;
 747
 748         if (p_dec->fmt_in.p_extra)
 749         {
 750             int i_ret = ExtradataInfoCreate(p_dec, CFSTR("avcC"),
 751                                             p_dec->fmt_in.p_extra,
 752                                             p_dec->fmt_in.i_extra);
 753             if (i_ret != VLC_SUCCESS)
 754                 return i_ret;
 755         }
 756         /* else: AnnexB case, we'll get extradata from first input blocks */
 757     }
 758     else if (p_sys->codec == kCMVideoCodecType_MPEG4Video)
 759     {
 760         if (!p_dec->fmt_in.i_extra)
 761             return VLC_EGENERIC;
 762         int i_ret = ESDSCreate(p_dec, p_dec->fmt_in.p_extra, p_dec->fmt_in.i_extra);
 763         if (i_ret != VLC_SUCCESS)
 764             return i_ret;
 765     }
 766     else
 767     {
 768         int i_ret = ExtradataInfoCreate(p_dec, NULL, NULL, 0);
 769         if (i_ret != VLC_SUCCESS)
 770             return i_ret;
 771     }
 772
 773     return VLC_SUCCESS;
 774 }
 775
 776 static int OpenDecoder(vlc_object_t *p_this)
 777 {
 778     decoder_t *p_dec = (decoder_t *)p_this;
 779
 780 #if TARGET_OS_IPHONE
 781     if (unlikely([[UIDevice currentDevice].systemVersion floatValue] < 8.0)) {
 782         msg_Warn(p_dec, "decoder skipped as OS is too old");
 783         return VLC_EGENERIC;
 784     }
 785 #endif
 786
 787     if (p_dec->fmt_in.i_cat != VIDEO_ES)
 788         return VLC_EGENERIC;
 789
 790     /* Fail if this module already failed to decode this ES */
 791     if (var_Type(p_dec, "videotoolbox-failed") != 0)
 792         return VLC_EGENERIC;
 793
 794     /* check quickly if we can digest the offered data */
 795     CMVideoCodecType codec;
 796     codec = CodecPrecheck(p_dec);
 797     if (codec == -1)
 798         return VLC_EGENERIC;
 799
 800     /* now that we see a chance to decode anything, allocate the
 801      * internals and start the decoding session */
 802     decoder_sys_t *p_sys;
 803     p_sys = malloc(sizeof(*p_sys));
 804     if (!p_sys)
 805         return VLC_ENOMEM;
 806     p_dec->p_sys = p_sys;
 807     p_sys->session = nil;
 808     p_sys->b_vt_feed = false;
 809     p_sys->b_vt_flush = false;
 810     p_sys->codec = codec;
 811     p_sys->videoFormatDescription = nil;
 812     p_sys->decoderConfiguration = nil;
 813     p_sys->destinationPixelBufferAttributes = nil;
 814     p_sys->extradataInfo = nil;
 815     p_sys->p_pic_reorder = NULL;
 816     p_sys->i_pic_reorder = 0;
 817     p_sys->i_pic_reorder_max = 4;
 818     p_sys->b_poc_based_reorder = false;
 819     p_sys->b_format_propagated = false;
 820     p_sys->b_abort = false;
 821     p_sys->b_enable_temporal_processing = false;
 822     h264_poc_context_init( &p_sys->pocctx );
 823     vlc_mutex_init(&p_sys->lock);
 824
 825     /* return our proper VLC internal state */
 826     p_dec->fmt_out.i_cat = p_dec->fmt_in.i_cat;
 827     p_dec->fmt_out.video = p_dec->fmt_in.video;
 828     if (!p_dec->fmt_out.video.i_sar_num || !p_dec->fmt_out.video.i_sar_den)
 829     {
 830         p_dec->fmt_out.video.i_sar_num = 1;
 831         p_dec->fmt_out.video.i_sar_den = 1;
 832     }
 833     if (!p_dec->fmt_out.video.i_visible_width
 834      || !p_dec->fmt_out.video.i_visible_height)
 835     {
 836         p_dec->fmt_out.video.i_visible_width = p_dec->fmt_out.video.i_width;
 837         p_dec->fmt_out.video.i_visible_height = p_dec->fmt_out.video.i_height;
 838     }
 839     else
 840     {
 841         p_dec->fmt_out.video.i_width = p_dec->fmt_out.video.i_visible_width;
 842         p_dec->fmt_out.video.i_height = p_dec->fmt_out.video.i_visible_height;
 843     }
 844
 845     p_dec->fmt_out.i_codec = 0;
 846
 847     if( codec == kCMVideoCodecType_H264 )
 848         p_sys->b_poc_based_reorder = true;
 849
 850     int i_ret = SetupDecoderExtradata(p_dec);
 851     if (i_ret != VLC_SUCCESS)
 852         goto error;
 853
 854     if (p_sys->extradataInfo != nil)
 855     {
 856         i_ret = StartVideoToolbox(p_dec);
 857         if (i_ret != VLC_SUCCESS)
 858             goto error;
 859     } /* else: late opening */
 860
 861     p_dec->pf_decode = DecodeBlock;
 862     p_dec->pf_flush  = Flush;
 863
 864     msg_Info(p_dec, "Using Video Toolbox to decode '%4.4s'", (char *)&p_dec->fmt_in.i_codec);
 865
 866     return VLC_SUCCESS;
 867
 868 error:
 869     CloseDecoder(p_this);
 870     return i_ret;
 871 }
 872
 873 static void CloseDecoder(vlc_object_t *p_this)
 874 {
 875     decoder_t *p_dec = (decoder_t *)p_this;
 876     decoder_sys_t *p_sys = p_dec->p_sys;
 877
 878     StopVideoToolbox(p_dec, true);
 879
 880     if (p_sys->codec == kCMVideoCodecType_H264)
 881         hxxx_helper_clean(&p_sys->hh);
 882
 883     vlc_mutex_destroy(&p_sys->lock);
 884     free(p_sys);
 885 }
 886
 887 #pragma mark - helpers
 888
 889 static BOOL deviceSupportsAdvancedProfiles()
 890 {
 891 #if TARGET_IPHONE_SIMULATOR
 892     return NO;
 893 #endif
 894 #if TARGET_OS_IPHONE
 895     size_t size;
 896     cpu_type_t type;
 897
 898     size = sizeof(type);
 899     sysctlbyname("hw.cputype", &type, &size, NULL, 0);
 900
 901     /* Support for H264 profile HIGH 10 was introduced with the first 64bit Apple ARM SoC, the A7 */
 902     if (type == CPU_TYPE_ARM64)
 903         return YES;
 904
 905     return NO;
 906 #else
 907     return NO;
 908 #endif
 909 }
 910
 911 static BOOL deviceSupportsAdvancedLevels()
 912 {
 913 #if TARGET_IPHONE_SIMULATOR
 914     return YES;
 915 #endif
 916 #if TARGET_OS_IPHONE
 917     size_t size;
 918     int32_t cpufamily;
 919
 920     size = sizeof(cpufamily);
 921     sysctlbyname("hw.cpufamily", &cpufamily, &size, NULL, 0);
 922
 923     /* Proper 4K decoding requires a Twister SoC
 924      * Everything below will kill the decoder daemon */
 925     if (cpufamily == CPUFAMILY_ARM_TWISTER) {
 926         return YES;
 927     }
 928
 929     return NO;
 930 #else
 931     return YES;
 932 #endif
 933 }
 934
 935 static inline void bo_add_mp4_tag_descr(bo_t *p_bo, uint8_t tag, uint32_t size)
 936 {
 937     bo_add_8(p_bo, tag);
 938     for (int i = 3; i>0; i--)
 939         bo_add_8(p_bo, (size>>(7*i)) | 0x80);
 940     bo_add_8(p_bo, size & 0x7F);
 941 }
 942
 943 static int ESDSCreate(decoder_t *p_dec, uint8_t *p_buf, uint32_t i_buf_size)
 944 {
 945     int full_size = 3 + 5 +13 + 5 + i_buf_size + 3;
 946     int config_size = 13 + 5 + i_buf_size;
 947     int padding = 12;
 948
 949     bo_t bo;
 950     bool status = bo_init(&bo, 1024);
 951     if (status != true)
 952         return VLC_EGENERIC;
 953
 954     bo_add_8(&bo, 0);       // Version
 955     bo_add_24be(&bo, 0);    // Flags
 956
 957     // elementary stream description tag
 958     bo_add_mp4_tag_descr(&bo, 0x03, full_size);
 959     bo_add_16be(&bo, 0);    // esid
 960     bo_add_8(&bo, 0);       // stream priority (0-3)
 961
 962     // decoder configuration description tag
 963     bo_add_mp4_tag_descr(&bo, 0x04, config_size);
 964     bo_add_8(&bo, 32);      // object type identification (32 == MPEG4)
 965     bo_add_8(&bo, 0x11);    // stream type
 966     bo_add_24be(&bo, 0);    // buffer size
 967     bo_add_32be(&bo, 0);    // max bitrate
 968     bo_add_32be(&bo, 0);    // avg bitrate
 969
 970     // decoder specific description tag
 971     bo_add_mp4_tag_descr(&bo, 0x05, i_buf_size);
 972     bo_add_mem(&bo, i_buf_size, p_buf);
 973
 974     // sync layer configuration description tag
 975     bo_add_8(&bo, 0x06);    // tag
 976     bo_add_8(&bo, 0x01);    // length
 977     bo_add_8(&bo, 0x02);    // no SL
 978
 979     int i_ret = ExtradataInfoCreate(p_dec, CFSTR("esds"), bo.b->p_buffer,
 980                                     bo.b->i_buffer);
 981     bo_deinit(&bo);
 982     return i_ret;
 983 }
 984
 985 static int avcCFromAnnexBCreate(decoder_t *p_dec)
 986 {
 987     decoder_sys_t *p_sys = p_dec->p_sys;
 988
 989     if (p_sys->hh.h264.i_sps_count == 0 || p_sys->hh.h264.i_pps_count == 0)
 990         return VLC_EGENERIC;
 991
 992     unsigned i_h264_width, i_h264_height, i_video_width, i_video_height;
 993     int i_sar_num, i_sar_den, i_ret;
 994     i_ret = h264_helper_get_current_picture_size(&p_sys->hh,
 995                                                  &i_h264_width, &i_h264_height,
 996                                                  &i_video_width, &i_video_height);
 997     if (i_ret != VLC_SUCCESS)
 998         return i_ret;
 999     i_ret = h264_helper_get_current_sar(&p_sys->hh, &i_sar_num, &i_sar_den);
1000     if (i_ret != VLC_SUCCESS)
1001         return i_ret;
1002
1003     p_dec->fmt_out.video.i_visible_width =
1004     p_dec->fmt_out.video.i_width = i_video_width;
1005     p_dec->fmt_out.video.i_visible_height =
1006     p_dec->fmt_out.video.i_height = i_video_height;
1007     p_dec->fmt_out.video.i_sar_num = i_sar_num;
1008     p_dec->fmt_out.video.i_sar_den = i_sar_den;
1009
1010     block_t *p_avcC = h264_helper_get_avcc_config(&p_sys->hh);
1011     if (!p_avcC)
1012         return VLC_EGENERIC;
1013
1014     i_ret = ExtradataInfoCreate(p_dec, CFSTR("avcC"), p_avcC->p_buffer,
1015                                 p_avcC->i_buffer);
1016     block_Release(p_avcC);
1017     return i_ret;
1018 }
1019
1020 static int ExtradataInfoCreate(decoder_t *p_dec, CFStringRef name, void *p_data,
1021                                size_t i_data)
1022 {
1023     decoder_sys_t *p_sys = p_dec->p_sys;
1024
1025     p_sys->extradataInfo =
1026         CFDictionaryCreateMutable(kCFAllocatorDefault, 1,
1027                                   &kCFTypeDictionaryKeyCallBacks,
1028                                   &kCFTypeDictionaryValueCallBacks);
1029     if (p_sys->extradataInfo == nil)
1030         return VLC_EGENERIC;
1031
1032     if (p_data == NULL)
1033         return VLC_SUCCESS;
1034
1035     CFDataRef extradata = CFDataCreate(kCFAllocatorDefault, p_data, i_data);
1036     if (extradata == nil)
1037     {
1038         CFRelease(p_sys->extradataInfo);
1039         p_sys->extradataInfo = nil;
1040         return VLC_EGENERIC;
1041     }
1042     CFDictionarySetValue(p_sys->extradataInfo, name, extradata);
1043     CFRelease(extradata);
1044     return VLC_SUCCESS;
1045 }
1046
1047 static CMSampleBufferRef VTSampleBufferCreate(decoder_t *p_dec,
1048                                               CMFormatDescriptionRef fmt_desc,
1049                                               block_t *p_block)
1050 {
1051     OSStatus status;
1052     CMBlockBufferRef  block_buf = NULL;
1053     CMSampleBufferRef sample_buf = NULL;
1054     CMTime pts;
1055     if(!p_dec->p_sys->b_poc_based_reorder && p_block->i_pts == VLC_TS_INVALID)
1056         pts = CMTimeMake(p_block->i_dts, CLOCK_FREQ);
1057     else
1058         pts = CMTimeMake(p_block->i_pts, CLOCK_FREQ);
1059
1060     CMSampleTimingInfo timeInfoArray[1] = { {
1061         .duration = CMTimeMake(p_block->i_length, 1),
1062         .presentationTimeStamp = pts,
1063         .decodeTimeStamp = CMTimeMake(p_block->i_dts, CLOCK_FREQ),
1064     } };
1065
1066     status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault,// structureAllocator
1067                                                 p_block->p_buffer,  // memoryBlock
1068                                                 p_block->i_buffer,  // blockLength
1069                                                 kCFAllocatorNull,   // blockAllocator
1070                                                 NULL,               // customBlockSource
1071                                                 0,                  // offsetToData
1072                                                 p_block->i_buffer,  // dataLength
1073                                                 false,              // flags
1074                                                 &block_buf);
1075
1076     if (!status) {
1077         status = CMSampleBufferCreate(kCFAllocatorDefault,  // allocator
1078                                       block_buf,            // dataBuffer
1079                                       TRUE,                 // dataReady
1080                                       0,                    // makeDataReadyCallback
1081                                       0,                    // makeDataReadyRefcon
1082                                       fmt_desc,             // formatDescription
1083                                       1,                    // numSamples
1084                                       1,                    // numSampleTimingEntries
1085                                       timeInfoArray,        // sampleTimingArray
1086                                       0,                    // numSampleSizeEntries
1087                                       NULL,                 // sampleSizeArray
1088                                       &sample_buf);
1089         if (status != noErr)
1090             msg_Warn(p_dec, "sample buffer creation failure %i", (int)status);
1091     } else
1092         msg_Warn(p_dec, "cm block buffer creation failure %i", (int)status);
1093
1094     if (block_buf != nil)
1095         CFRelease(block_buf);
1096     block_buf = nil;
1097
1098     return sample_buf;
1099 }
1100
1101 void VTDictionarySetInt32(CFMutableDictionaryRef dict, CFStringRef key, int value)
1102 {
1103     CFNumberRef number;
1104     number = CFNumberCreate(NULL, kCFNumberSInt32Type, &value);
1105     CFDictionarySetValue(dict, key, number);
1106     CFRelease(number);
1107 }
1108
1109 static void copy420YpCbCr8Planar(picture_t *p_pic,
1110                                  CVPixelBufferRef buffer,
1111                                  unsigned i_width,
1112                                  unsigned i_height)
1113 {
1114     uint8_t *pp_plane[2];
1115     size_t pi_pitch[2];
1116
1117     if (!buffer || i_width == 0 || i_height == 0)
1118         return;
1119
1120     CVPixelBufferLockBaseAddress(buffer, kCVPixelBufferLock_ReadOnly);
1121
1122     for (int i = 0; i < 2; i++) {
1123         pp_plane[i] = CVPixelBufferGetBaseAddressOfPlane(buffer, i);
1124         pi_pitch[i] = CVPixelBufferGetBytesPerRowOfPlane(buffer, i);
1125     }
1126
1127     CopyFromNv12ToI420(p_pic, pp_plane, pi_pitch, i_height);
1128
1129     CVPixelBufferUnlockBaseAddress(buffer, kCVPixelBufferLock_ReadOnly);
1130 }
1131
1132 static int HandleVTStatus(decoder_t *p_dec, OSStatus status)
1133 {
1134 #define VTERRCASE(x) \
1135     case x: msg_Err(p_dec, "vt session error: '" #x "'"); break;
1136
1137     switch (status)
1138     {
1139         case 0:
1140             return VLC_SUCCESS;
1141
1142         VTERRCASE(kVTPropertyNotSupportedErr)
1143         VTERRCASE(kVTPropertyReadOnlyErr)
1144         VTERRCASE(kVTParameterErr)
1145         VTERRCASE(kVTInvalidSessionErr)
1146         VTERRCASE(kVTAllocationFailedErr)
1147         VTERRCASE(kVTPixelTransferNotSupportedErr)
1148         VTERRCASE(kVTCouldNotFindVideoDecoderErr)
1149         VTERRCASE(kVTCouldNotCreateInstanceErr)
1150         VTERRCASE(kVTCouldNotFindVideoEncoderErr)
1151         VTERRCASE(kVTVideoDecoderBadDataErr)
1152         VTERRCASE(kVTVideoDecoderUnsupportedDataFormatErr)
1153         VTERRCASE(kVTVideoDecoderMalfunctionErr)
1154         VTERRCASE(kVTVideoEncoderMalfunctionErr)
1155         VTERRCASE(kVTVideoDecoderNotAvailableNowErr)
1156         VTERRCASE(kVTImageRotationNotSupportedErr)
1157         VTERRCASE(kVTVideoEncoderNotAvailableNowErr)
1158         VTERRCASE(kVTFormatDescriptionChangeNotSupportedErr)
1159         VTERRCASE(kVTInsufficientSourceColorDataErr)
1160         VTERRCASE(kVTCouldNotCreateColorCorrectionDataErr)
1161         VTERRCASE(kVTColorSyncTransformConvertFailedErr)
1162         VTERRCASE(kVTVideoDecoderAuthorizationErr)
1163         VTERRCASE(kVTVideoEncoderAuthorizationErr)
1164         VTERRCASE(kVTColorCorrectionPixelTransferFailedErr)
1165         VTERRCASE(kVTMultiPassStorageIdentifierMismatchErr)
1166         VTERRCASE(kVTMultiPassStorageInvalidErr)
1167         VTERRCASE(kVTFrameSiloInvalidTimeStampErr)
1168         VTERRCASE(kVTFrameSiloInvalidTimeRangeErr)
1169         VTERRCASE(kVTCouldNotFindTemporalFilterErr)
1170         VTERRCASE(kVTPixelTransferNotPermittedErr)
1171         VTERRCASE(kVTColorCorrectionImageRotationFailedErr)
1172
1173         default:
1174             msg_Err(p_dec, "unknown vt session error (%i)", (int)status);
1175     }
1176 #undef VTERRCASE
1177     return VLC_EGENERIC;
1178 }
1179
1180 #pragma mark - actual decoding
1181
1182 static void Flush(decoder_t *p_dec)
1183 {
1184     decoder_sys_t *p_sys = p_dec->p_sys;
1185
1186     /* There is no Flush in VT api, ask to restart VT from next DecodeBlock if
1187      * we already feed some input blocks (it's better to not restart here in
1188      * order to avoid useless restart just before a close). */
1189     p_sys->b_vt_flush = p_sys->b_vt_feed;
1190 }
1191
1192 static void Drain(decoder_t *p_dec)
1193 {
1194     decoder_sys_t *p_sys = p_dec->p_sys;
1195
1196     /* draining: return last pictures of the reordered queue */
1197     if (p_sys->session)
1198         VTDecompressionSessionWaitForAsynchronousFrames(p_sys->session);
1199
1200     vlc_mutex_lock(&p_sys->lock);
1201     FlushDPB( p_dec );
1202     vlc_mutex_unlock(&p_sys->lock);
1203 }
1204
1205 static int DecodeBlock(decoder_t *p_dec, block_t *p_block)
1206 {
1207     decoder_sys_t *p_sys = p_dec->p_sys;
1208     frame_info_t *p_info = NULL;
1209
1210     if (p_sys->b_vt_flush) {
1211         RestartVideoToolbox(p_dec, false);
1212         p_sys->b_vt_flush = false;
1213     }
1214
1215     if (p_block == NULL)
1216     {
1217         Drain(p_dec);
1218         return VLCDEC_SUCCESS;
1219     }
1220
1221     vlc_mutex_lock(&p_sys->lock);
1222     if (p_sys->b_abort) { /* abort from output thread (DecoderCallback) */
1223         vlc_mutex_unlock(&p_sys->lock);
1224         goto reload;
1225     }
1226     vlc_mutex_unlock(&p_sys->lock);
1227
1228     if (unlikely(p_block->i_flags&(BLOCK_FLAG_CORRUPTED)))
1229     {
1230         if (p_sys->b_vt_feed)
1231         {
1232             Drain(p_dec);
1233             RestartVideoToolbox(p_dec, false);
1234         }
1235         goto skip;
1236     }
1237
1238     bool b_config_changed = false;
1239     if (p_sys->codec == kCMVideoCodecType_H264 && p_sys->hh.pf_process_block)
1240     {
1241         p_block = p_sys->hh.pf_process_block(&p_sys->hh, p_block, &b_config_changed);
1242         if (!p_block)
1243             return VLCDEC_SUCCESS;
1244     }
1245
1246     if (b_config_changed)
1247     {
1248         /* decoding didn't start yet, which is ok for H264, let's see
1249          * if we can use this block to get going */
1250         assert(p_sys->codec == kCMVideoCodecType_H264 && !p_sys->hh.b_is_xvcC);
1251         if (p_sys->session)
1252         {
1253             msg_Dbg(p_dec, "SPS/PPS changed: draining H264 decoder");
1254             Drain(p_dec);
1255             msg_Dbg(p_dec, "SPS/PPS changed: restarting H264 decoder");
1256             StopVideoToolbox(p_dec, true);
1257         }
1258
1259         int i_ret = avcCFromAnnexBCreate(p_dec);
1260         if (i_ret == VLC_SUCCESS)
1261         {
1262             if ((p_block->i_flags & BLOCK_FLAG_TOP_FIELD_FIRST
1263              || p_block->i_flags & BLOCK_FLAG_BOTTOM_FIELD_FIRST)
1264              && var_InheritBool(p_dec, "videotoolbox-temporal-deinterlacing"))
1265                 p_sys->b_enable_temporal_processing = true;
1266
1267             msg_Dbg(p_dec, "Got SPS/PPS: late opening of H264 decoder");
1268             StartVideoToolbox(p_dec);
1269         }
1270         if (!p_sys->session)
1271             goto skip;
1272     }
1273
1274
1275     p_info = CreateReorderInfo(p_sys, p_block);
1276     if(unlikely(!p_info))
1277         goto skip;
1278
1279     CMSampleBufferRef sampleBuffer =
1280         VTSampleBufferCreate(p_dec, p_sys->videoFormatDescription, p_block);
1281     if (unlikely(!sampleBuffer))
1282         goto skip;
1283
1284     VTDecodeInfoFlags flagOut;
1285     VTDecodeFrameFlags decoderFlags = kVTDecodeFrame_EnableAsynchronousDecompression;
1286     if (unlikely(p_sys->b_enable_temporal_processing))
1287         decoderFlags |= kVTDecodeFrame_EnableTemporalProcessing;
1288
1289     OSStatus status =
1290         VTDecompressionSessionDecodeFrame(p_sys->session, sampleBuffer,
1291                                           decoderFlags, p_info, &flagOut);
1292     if (HandleVTStatus(p_dec, status) == VLC_SUCCESS)
1293     {
1294         p_sys->b_vt_feed = true;
1295         p_info = NULL;
1296     }
1297     else
1298     {
1299         switch (status)
1300         {
1301             case kCVReturnInvalidArgument:
1302                 msg_Err(p_dec, "decoder failure: invalid argument");
1303                 /* The decoder module will be reloaded next time since we already
1304                  * modified the input block */
1305                 vlc_mutex_lock(&p_sys->lock);
1306                 p_dec->p_sys->b_abort = true;
1307                 vlc_mutex_unlock(&p_sys->lock);
1308                 break;
1309             case -8969 /* codecBadDataErr */:
1310             case kVTVideoDecoderBadDataErr:
1311                 if (RestartVideoToolbox(p_dec, true) == VLC_SUCCESS)
1312                 {
1313                     status = VTDecompressionSessionDecodeFrame(p_sys->session,
1314                                     sampleBuffer, decoderFlags, p_info, &flagOut);
1315
1316                     if (status != 0)
1317                     {
1318                         free( p_info );
1319                         StopVideoToolbox(p_dec, true);
1320                     }
1321                 }
1322                 break;
1323             case -8960 /* codecErr */:
1324             case kVTVideoDecoderMalfunctionErr:
1325             case kVTInvalidSessionErr:
1326                 RestartVideoToolbox(p_dec, true);
1327                 break;
1328         }
1329         p_info = NULL;
1330     }
1331     CFRelease(sampleBuffer);
1332
1333 skip:
1334     free(p_info);
1335     block_Release(p_block);
1336     return VLCDEC_SUCCESS;
1337
1338 reload:
1339     /* Add an empty variable so that videotoolbox won't be loaded again for
1340      * this ES */
1341     var_Create(p_dec, "videotoolbox-failed", VLC_VAR_VOID);
1342     return VLCDEC_RELOAD;
1343 }
1344
1345 static int UpdateVideoFormat(decoder_t *p_dec, CVPixelBufferRef imageBuffer)
1346 {
1347     CFDictionaryRef attachments = CVBufferGetAttachments(imageBuffer, kCVAttachmentMode_ShouldPropagate);
1348     NSDictionary *attachmentDict = (NSDictionary *)attachments;
1349 #ifndef NDEBUG
1350     NSLog(@"%@", attachments);
1351 #endif
1352     if (attachmentDict == nil || attachmentDict.count == 0)
1353         return -1;
1354
1355     NSString *colorSpace = attachmentDict[(NSString *)kCVImageBufferYCbCrMatrixKey];
1356     if (colorSpace != nil) {
1357         if ([colorSpace isEqualToString:(NSString *)kCVImageBufferYCbCrMatrix_ITU_R_601_4])
1358             p_dec->fmt_out.video.space = COLOR_SPACE_BT601;
1359         else if ([colorSpace isEqualToString:(NSString *)kCVImageBufferYCbCrMatrix_ITU_R_709_2])
1360             p_dec->fmt_out.video.space = COLOR_SPACE_BT709;
1361         else
1362             p_dec->fmt_out.video.space = COLOR_SPACE_UNDEF;
1363     }
1364
1365     NSString *colorprimary = attachmentDict[(NSString *)kCVImageBufferColorPrimariesKey];
1366     if (colorprimary != nil) {
1367         if ([colorprimary isEqualToString:(NSString *)kCVImageBufferColorPrimaries_SMPTE_C] ||
1368             [colorprimary isEqualToString:(NSString *)kCVImageBufferColorPrimaries_EBU_3213])
1369             p_dec->fmt_out.video.primaries = COLOR_PRIMARIES_BT601_625;
1370         else if ([colorprimary isEqualToString:(NSString *)kCVImageBufferColorPrimaries_ITU_R_709_2])
1371             p_dec->fmt_out.video.primaries = COLOR_PRIMARIES_BT709;
1372         else if ([colorprimary isEqualToString:(NSString *)kCVImageBufferColorPrimaries_P22])
1373             p_dec->fmt_out.video.primaries = COLOR_PRIMARIES_DCI_P3;
1374         else
1375             p_dec->fmt_out.video.primaries = COLOR_PRIMARIES_UNDEF;
1376     }
1377
1378     NSString *transfer = attachmentDict[(NSString *)kCVImageBufferTransferFunctionKey];
1379     if (transfer != nil) {
1380         if ([transfer isEqualToString:(NSString *)kCVImageBufferTransferFunction_ITU_R_709_2] ||
1381             [transfer isEqualToString:(NSString *)kCVImageBufferTransferFunction_SMPTE_240M_1995])
1382             p_dec->fmt_out.video.transfer = TRANSFER_FUNC_BT709;
1383         else
1384             p_dec->fmt_out.video.transfer = TRANSFER_FUNC_UNDEF;
1385     }
1386
1387     NSString *chromaLocation = attachmentDict[(NSString *)kCVImageBufferChromaLocationTopFieldKey];
1388     if (chromaLocation != nil) {
1389         if ([chromaLocation isEqualToString:(NSString *)kCVImageBufferChromaLocation_Left] ||
1390             [chromaLocation isEqualToString:(NSString *)kCVImageBufferChromaLocation_DV420])
1391             p_dec->fmt_out.video.chroma_location = CHROMA_LOCATION_LEFT;
1392         else if ([chromaLocation isEqualToString:(NSString *)kCVImageBufferChromaLocation_Center])
1393             p_dec->fmt_out.video.chroma_location = CHROMA_LOCATION_CENTER;
1394         else if ([chromaLocation isEqualToString:(NSString *)kCVImageBufferChromaLocation_TopLeft])
1395             p_dec->fmt_out.video.chroma_location = CHROMA_LOCATION_TOP_LEFT;
1396         else if ([chromaLocation isEqualToString:(NSString *)kCVImageBufferChromaLocation_Top])
1397             p_dec->fmt_out.video.chroma_location = CHROMA_LOCATION_TOP_CENTER;
1398         else
1399             p_dec->fmt_out.video.chroma_location = CHROMA_LOCATION_UNDEF;
1400     }
1401     if (p_dec->fmt_out.video.chroma_location == CHROMA_LOCATION_UNDEF) {
1402         chromaLocation = attachmentDict[(NSString *)kCVImageBufferChromaLocationBottomFieldKey];
1403         if (chromaLocation != nil) {
1404             if ([chromaLocation isEqualToString:(NSString *)kCVImageBufferChromaLocation_BottomLeft])
1405                 p_dec->fmt_out.video.chroma_location = CHROMA_LOCATION_BOTTOM_LEFT;
1406             else if ([chromaLocation isEqualToString:(NSString *)kCVImageBufferChromaLocation_Bottom])
1407                 p_dec->fmt_out.video.chroma_location = CHROMA_LOCATION_BOTTOM_CENTER;
1408         }
1409     }
1410
1411     uint32_t cvfmt = CVPixelBufferGetPixelFormatType(imageBuffer);
1412     switch (cvfmt)
1413     {
1414         case kCVPixelFormatType_422YpCbCr8:
1415         case 'yuv2':
1416             p_dec->fmt_out.i_codec = VLC_CODEC_CVPX_UYVY;
1417             assert(CVPixelBufferIsPlanar(imageBuffer) == false);
1418             break;
1419         case kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange:
1420         case kCVPixelFormatType_420YpCbCr8BiPlanarFullRange:
1421             p_dec->fmt_out.i_codec = VLC_CODEC_CVPX_NV12;
1422             assert(CVPixelBufferIsPlanar(imageBuffer) == true);
1423             break;
1424         case kCVPixelFormatType_420YpCbCr8Planar:
1425             p_dec->fmt_out.i_codec = VLC_CODEC_CVPX_I420;
1426             assert(CVPixelBufferIsPlanar(imageBuffer) == true);
1427             break;
1428         case kCVPixelFormatType_32BGRA:
1429             p_dec->fmt_out.i_codec = VLC_CODEC_CVPX_BGRA;
1430             assert(CVPixelBufferIsPlanar(imageBuffer) == false);
1431             break;
1432         default:
1433             p_dec->p_sys->b_abort = true;
1434             return -1;
1435     }
1436     return decoder_UpdateVideoFormat(p_dec);
1437 }
1438
1439 static void DecoderCallback(void *decompressionOutputRefCon,
1440                             void *sourceFrameRefCon,
1441                             OSStatus status,
1442                             VTDecodeInfoFlags infoFlags,
1443                             CVPixelBufferRef imageBuffer,
1444                             CMTime pts,
1445                             CMTime duration)
1446 {
1447     VLC_UNUSED(duration);
1448     decoder_t *p_dec = (decoder_t *)decompressionOutputRefCon;
1449     decoder_sys_t *p_sys = p_dec->p_sys;
1450     frame_info_t *p_info = (frame_info_t *) sourceFrameRefCon;
1451
1452     if (status != noErr) {
1453         msg_Warn(p_dec, "decoding of a frame failed (%i, %u)", (int)status, (unsigned int) infoFlags);
1454         if( status != kVTVideoDecoderBadDataErr && status != -8969 )
1455             free(p_info);
1456         return;
1457     }
1458     assert(imageBuffer);
1459
1460     if (unlikely(!p_sys->b_format_propagated)) {
1461         vlc_mutex_lock(&p_sys->lock);
1462         p_sys->b_format_propagated =
1463             UpdateVideoFormat(p_dec, imageBuffer) == VLC_SUCCESS;
1464         vlc_mutex_unlock(&p_sys->lock);
1465
1466         if (!p_sys->b_format_propagated)
1467         {
1468             free(p_info);
1469             return;
1470         }
1471         assert(p_dec->fmt_out.i_codec != 0);
1472     }
1473
1474     if (infoFlags & kVTDecodeInfo_FrameDropped)
1475     {
1476         printf("OUT %ld\n", pts.value);
1477         msg_Dbg(p_dec, "decoder dropped frame");
1478         free(p_info);
1479         return;
1480     }
1481
1482     if (!CMTIME_IS_VALID(pts))
1483     {
1484         free(p_info);
1485         return;
1486     }
1487
1488     if (CVPixelBufferGetDataSize(imageBuffer) == 0)
1489     {
1490         free(p_info);
1491         return;
1492     }
1493
1494     if(likely(p_info))
1495     {
1496         picture_t *p_pic = decoder_NewPicture(p_dec);
1497         if (!p_pic)
1498         {
1499             free(p_info);
1500             return;
1501         }
1502
1503         if (unlikely(!p_pic->p_sys)) {
1504             vlc_mutex_lock(&p_sys->lock);
1505             p_dec->p_sys->b_abort = true;
1506             vlc_mutex_unlock(&p_sys->lock);
1507             picture_Release(p_pic);
1508             free(p_info);
1509             return;
1510         }
1511
1512         /* Can happen if the pic was discarded */
1513         if (p_pic->p_sys->pixelBuffer != nil)
1514             CFRelease(p_pic->p_sys->pixelBuffer);
1515
1516         /* will be freed by the vout */
1517         p_pic->p_sys->pixelBuffer = CVPixelBufferRetain(imageBuffer);
1518
1519         p_info->p_picture = p_pic;
1520
1521         p_pic->date = pts.value;
1522         p_pic->b_progressive = p_info->b_progressive;
1523
1524         vlc_mutex_lock(&p_sys->lock);
1525         OnDecodedFrame( p_dec, p_info );
1526         vlc_mutex_unlock(&p_sys->lock);
1527     }
1528
1529     return;
1530 }