modules/packetizer/h264.c

   1 /*****************************************************************************
   2  * h264.c: h264/avc video packetizer
   3  *****************************************************************************
   4  * Copyright (C) 2001, 2002, 2006 VLC authors and VideoLAN
   5  * $Id$
   6  *
   7  * Authors: Laurent Aimar <fenrir@via.ecp.fr>
   8  *          Eric Petit <titer@videolan.org>
   9  *          Gildas Bazin <gbazin@videolan.org>
  10  *          Derk-Jan Hartman <hartman at videolan dot org>
  11  *
  12  * This program is free software; you can redistribute it and/or modify it
  13  * under the terms of the GNU Lesser General Public License as published by
  14  * the Free Software Foundation; either version 2.1 of the License, or
  15  * (at your option) any later version.
  16  *
  17  * This program is distributed in the hope that it will be useful,
  18  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  19  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  20  * GNU Lesser General Public License for more details.
  21  *
  22  * You should have received a copy of the GNU Lesser General Public License
  23  * along with this program; if not, write to the Free Software Foundation,
  24  * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  25  *****************************************************************************/
  26
  27 /*****************************************************************************
  28  * Preamble
  29  *****************************************************************************/
  30
  31 #ifdef HAVE_CONFIG_H
  32 # include "config.h"
  33 #endif
  34
  35 #include <vlc_common.h>
  36 #include <vlc_plugin.h>
  37 #include <vlc_sout.h>
  38 #include <vlc_codec.h>
  39 #include <vlc_block.h>
  40
  41 #include <vlc_block_helper.h>
  42 #include <vlc_bits.h>
  43 #include "h264_nal.h"
  44 #include "h264_slice.h"
  45 #include "hxxx_nal.h"
  46 #include "hxxx_sei.h"
  47 #include "hxxx_common.h"
  48 #include "packetizer_helper.h"
  49 #include "startcode_helper.h"
  50
  51 #include <limits.h>
  52
  53 /*****************************************************************************
  54  * Module descriptor
  55  *****************************************************************************/
  56 static int  Open ( vlc_object_t * );
  57 static void Close( vlc_object_t * );
  58
  59 vlc_module_begin ()
  60     set_category( CAT_SOUT )
  61     set_subcategory( SUBCAT_SOUT_PACKETIZER )
  62     set_description( N_("H.264 video packetizer") )
  63     set_capability( "packetizer", 50 )
  64     set_callbacks( Open, Close )
  65 vlc_module_end ()
  66
  67
  68 /****************************************************************************
  69  * Local prototypes
  70  ****************************************************************************/
  71
  72 struct decoder_sys_t
  73 {
  74     /* */
  75     packetizer_t packetizer;
  76
  77     /* */
  78     bool    b_slice;
  79     block_t *p_frame;
  80     block_t **pp_frame_last;
  81     block_t *p_sei;
  82     block_t **pp_sei_last;
  83     /* a new sps/pps can be transmitted outside of iframes */
  84     bool    b_new_sps;
  85     bool    b_new_pps;
  86
  87     struct
  88     {
  89         block_t *p_block;
  90         h264_sequence_parameter_set_t *p_sps;
  91     } sps[H264_SPS_ID_MAX + 1];
  92     struct
  93     {
  94         block_t *p_block;
  95         h264_picture_parameter_set_t *p_pps;
  96     } pps[H264_PPS_ID_MAX + 1];
  97     const h264_sequence_parameter_set_t *p_active_sps;
  98     const h264_picture_parameter_set_t *p_active_pps;
  99
 100     /* avcC data */
 101     uint8_t i_avcC_length_size;
 102
 103     /* From SEI for current frame */
 104     uint8_t i_pic_struct;
 105     uint8_t i_dpb_output_delay;
 106     unsigned i_recovery_frame_cnt;
 107
 108     /* Useful values of the Slice Header */
 109     h264_slice_t slice;
 110
 111     /* */
 112     int i_next_block_flags;
 113     bool b_recovered;
 114     unsigned i_recoveryfnum;
 115
 116     /* POC */
 117     poc_context_t pocctx;
 118     struct
 119     {
 120         mtime_t pts;
 121         int num;
 122     } prevdatedpoc;
 123
 124     mtime_t i_frame_pts;
 125     mtime_t i_frame_dts;
 126
 127     date_t dts;
 128
 129     /* */
 130     cc_storage_t *p_ccs;
 131 };
 132
 133 #define BLOCK_FLAG_PRIVATE_AUD (1 << BLOCK_FLAG_PRIVATE_SHIFT)
 134 #define BLOCK_FLAG_DROP        (2 << BLOCK_FLAG_PRIVATE_SHIFT)
 135
 136 static block_t *Packetize( decoder_t *, block_t ** );
 137 static block_t *PacketizeAVC1( decoder_t *, block_t ** );
 138 static block_t *GetCc( decoder_t *p_dec, bool pb_present[4], int * );
 139 static void PacketizeFlush( decoder_t * );
 140
 141 static void PacketizeReset( void *p_private, bool b_broken );
 142 static block_t *PacketizeParse( void *p_private, bool *pb_ts_used, block_t * );
 143 static int PacketizeValidate( void *p_private, block_t * );
 144
 145 static block_t *ParseNALBlock( decoder_t *, bool *pb_ts_used, block_t * );
 146
 147 static block_t *OutputPicture( decoder_t *p_dec );
 148 static void PutSPS( decoder_t *p_dec, block_t *p_frag );
 149 static void PutPPS( decoder_t *p_dec, block_t *p_frag );
 150 static bool ParseSliceHeader( decoder_t *p_dec, const block_t *p_frag, h264_slice_t *p_slice );
 151 static bool ParseSeiCallback( const hxxx_sei_data_t *, void * );
 152
 153
 154 static const uint8_t p_h264_startcode[3] = { 0x00, 0x00, 0x01 };
 155
 156 /*****************************************************************************
 157  * Helpers
 158  *****************************************************************************/
 159
 160 static void StoreSPS( decoder_sys_t *p_sys, uint8_t i_id,
 161                       block_t *p_block, h264_sequence_parameter_set_t *p_sps )
 162 {
 163     if( p_sys->sps[i_id].p_block )
 164         block_Release( p_sys->sps[i_id].p_block );
 165     if( p_sys->sps[i_id].p_sps )
 166         h264_release_sps( p_sys->sps[i_id].p_sps );
 167     if( p_sys->sps[i_id].p_sps == p_sys->p_active_sps )
 168         p_sys->p_active_sps = NULL;
 169     p_sys->sps[i_id].p_block = p_block;
 170     p_sys->sps[i_id].p_sps = p_sps;
 171 }
 172
 173 static void StorePPS( decoder_sys_t *p_sys, uint8_t i_id,
 174                       block_t *p_block, h264_picture_parameter_set_t *p_pps )
 175 {
 176     if( p_sys->pps[i_id].p_block )
 177         block_Release( p_sys->pps[i_id].p_block );
 178     if( p_sys->pps[i_id].p_pps )
 179         h264_release_pps( p_sys->pps[i_id].p_pps );
 180     if( p_sys->pps[i_id].p_pps == p_sys->p_active_pps )
 181         p_sys->p_active_pps = NULL;
 182     p_sys->pps[i_id].p_block = p_block;
 183     p_sys->pps[i_id].p_pps = p_pps;
 184 }
 185
 186 static void ActivateSets( decoder_t *p_dec, const h264_sequence_parameter_set_t *p_sps,
 187                                             const h264_picture_parameter_set_t *p_pps )
 188 {
 189     decoder_sys_t *p_sys = p_dec->p_sys;
 190
 191     p_sys->p_active_pps = p_pps;
 192     p_sys->p_active_sps = p_sps;
 193
 194     if( p_sps )
 195     {
 196         p_dec->fmt_out.i_profile = p_sps->i_profile;
 197         p_dec->fmt_out.i_level = p_sps->i_level;
 198
 199         (void) h264_get_picture_size( p_sps, &p_dec->fmt_out.video.i_width,
 200                                       &p_dec->fmt_out.video.i_height,
 201                                       &p_dec->fmt_out.video.i_visible_width,
 202                                       &p_dec->fmt_out.video.i_visible_height );
 203
 204         if( p_sps->vui.i_sar_num != 0 && p_sps->vui.i_sar_den != 0 )
 205         {
 206             p_dec->fmt_out.video.i_sar_num = p_sps->vui.i_sar_num;
 207             p_dec->fmt_out.video.i_sar_den = p_sps->vui.i_sar_den;
 208         }
 209
 210         if( p_sps->vui.b_valid )
 211         {
 212             if( !p_dec->fmt_in.video.i_frame_rate_base && p_sps->vui.i_num_units_in_tick > 0 )
 213             {
 214                 const unsigned i_rate_base = p_sps->vui.i_num_units_in_tick;
 215                 const unsigned i_rate = p_sps->vui.i_time_scale >> 1; /* num_clock_ts == 2 */
 216                 if( i_rate_base != p_dec->fmt_out.video.i_frame_rate_base ||
 217                     i_rate != p_dec->fmt_out.video.i_frame_rate )
 218                 {
 219                     p_dec->fmt_out.video.i_frame_rate_base = i_rate_base;
 220                     p_dec->fmt_out.video.i_frame_rate = i_rate;
 221                     date_Change( &p_sys->dts, p_sps->vui.i_time_scale, p_sps->vui.i_num_units_in_tick );
 222                 }
 223             }
 224             if( p_dec->fmt_in.video.primaries == COLOR_PRIMARIES_UNDEF )
 225             {
 226                 p_dec->fmt_out.video.primaries =
 227                         hxxx_colour_primaries_to_vlc( p_sps->vui.colour.i_colour_primaries );
 228                 p_dec->fmt_out.video.transfer =
 229                         hxxx_transfer_characteristics_to_vlc( p_sps->vui.colour.i_transfer_characteristics );
 230                 p_dec->fmt_out.video.space =
 231                         hxxx_matrix_coeffs_to_vlc( p_sps->vui.colour.i_matrix_coefficients );
 232                 p_dec->fmt_out.video.b_color_range_full = p_sps->vui.colour.b_full_range;
 233             }
 234         }
 235     }
 236 }
 237
 238 static bool IsFirstVCLNALUnit( const h264_slice_t *p_prev, const h264_slice_t *p_cur )
 239 {
 240     /* Detection of the first VCL NAL unit of a primary coded picture
 241      * (cf. 7.4.1.2.4) */
 242     if( p_cur->i_frame_num != p_prev->i_frame_num ||
 243         p_cur->i_pic_parameter_set_id != p_prev->i_pic_parameter_set_id ||
 244         p_cur->i_field_pic_flag != p_prev->i_field_pic_flag ||
 245        !p_cur->i_nal_ref_idc != !p_prev->i_nal_ref_idc )
 246         return true;
 247     if( (p_cur->i_bottom_field_flag != -1) &&
 248         (p_prev->i_bottom_field_flag != -1) &&
 249         (p_cur->i_bottom_field_flag != p_prev->i_bottom_field_flag) )
 250         return true;
 251     if( p_cur->i_pic_order_cnt_type == 0 &&
 252        ( p_cur->i_pic_order_cnt_lsb != p_prev->i_pic_order_cnt_lsb ||
 253          p_cur->i_delta_pic_order_cnt_bottom != p_prev->i_delta_pic_order_cnt_bottom ) )
 254         return true;
 255     else if( p_cur->i_pic_order_cnt_type == 1 &&
 256            ( p_cur->i_delta_pic_order_cnt0 != p_prev->i_delta_pic_order_cnt0 ||
 257              p_cur->i_delta_pic_order_cnt1 != p_prev->i_delta_pic_order_cnt1 ) )
 258         return true;
 259     if( ( p_cur->i_nal_type == H264_NAL_SLICE_IDR || p_prev->i_nal_type == H264_NAL_SLICE_IDR ) &&
 260         ( p_cur->i_nal_type != p_prev->i_nal_type || p_cur->i_idr_pic_id != p_prev->i_idr_pic_id ) )
 261         return true;
 262
 263     return false;
 264 }
 265
 266 static void DropStoredNAL( decoder_sys_t *p_sys )
 267 {
 268     block_ChainRelease( p_sys->p_frame );
 269     block_ChainRelease( p_sys->p_sei );
 270     p_sys->p_frame = NULL;
 271     p_sys->pp_frame_last = &p_sys->p_frame;
 272     p_sys->p_sei = NULL;
 273     p_sys->pp_sei_last = &p_sys->p_sei;
 274 }
 275
 276 /*****************************************************************************
 277  * Open: probe the packetizer and return score
 278  * When opening after demux, the packetizer is only loaded AFTER the decoder
 279  * That means that what you set in fmt_out is ignored by the decoder in this special case
 280  *****************************************************************************/
 281 static int Open( vlc_object_t *p_this )
 282 {
 283     decoder_t     *p_dec = (decoder_t*)p_this;
 284     decoder_sys_t *p_sys;
 285     int i;
 286
 287     const bool b_avc = (p_dec->fmt_in.i_original_fourcc == VLC_FOURCC( 'a', 'v', 'c', '1' ));
 288
 289     if( p_dec->fmt_in.i_codec != VLC_CODEC_H264 )
 290         return VLC_EGENERIC;
 291     if( b_avc && p_dec->fmt_in.i_extra < 7 )
 292         return VLC_EGENERIC;
 293
 294     /* Allocate the memory needed to store the decoder's structure */
 295     if( ( p_dec->p_sys = p_sys = malloc( sizeof(decoder_sys_t) ) ) == NULL )
 296     {
 297         return VLC_ENOMEM;
 298     }
 299
 300     p_sys->p_ccs = cc_storage_new();
 301     if( unlikely(!p_sys->p_ccs) )
 302     {
 303         free( p_dec->p_sys );
 304         return VLC_ENOMEM;
 305     }
 306
 307     packetizer_Init( &p_sys->packetizer,
 308                      p_h264_startcode, sizeof(p_h264_startcode), startcode_FindAnnexB,
 309                      p_h264_startcode, 1, 5,
 310                      PacketizeReset, PacketizeParse, PacketizeValidate, p_dec );
 311
 312     p_sys->b_slice = false;
 313     p_sys->p_frame = NULL;
 314     p_sys->pp_frame_last = &p_sys->p_frame;
 315     p_sys->p_sei = NULL;
 316     p_sys->pp_sei_last = &p_sys->p_sei;
 317     p_sys->b_new_sps = false;
 318     p_sys->b_new_pps = false;
 319
 320     for( i = 0; i <= H264_SPS_ID_MAX; i++ )
 321     {
 322         p_sys->sps[i].p_sps = NULL;
 323         p_sys->sps[i].p_block = NULL;
 324     }
 325     p_sys->p_active_sps = NULL;
 326     for( i = 0; i <= H264_PPS_ID_MAX; i++ )
 327     {
 328         p_sys->pps[i].p_pps = NULL;
 329         p_sys->pps[i].p_block = NULL;
 330     }
 331     p_sys->p_active_pps = NULL;
 332     p_sys->i_recovery_frame_cnt = UINT_MAX;
 333
 334     h264_slice_init( &p_sys->slice );
 335
 336     p_sys->i_next_block_flags = 0;
 337     p_sys->b_recovered = false;
 338     p_sys->i_recoveryfnum = UINT_MAX;
 339     p_sys->i_frame_dts = VLC_TS_INVALID;
 340     p_sys->i_frame_pts = VLC_TS_INVALID;
 341     p_sys->i_dpb_output_delay = 0;
 342
 343     /* POC */
 344     h264_poc_context_init( &p_sys->pocctx );
 345     p_sys->prevdatedpoc.pts = VLC_TS_INVALID;
 346
 347     date_Init( &p_sys->dts, 30000 * 2, 1001 );
 348     date_Set( &p_sys->dts, VLC_TS_INVALID );
 349
 350     /* Setup properties */
 351     es_format_Copy( &p_dec->fmt_out, &p_dec->fmt_in );
 352     p_dec->fmt_out.i_codec = VLC_CODEC_H264;
 353     p_dec->fmt_out.b_packetized = true;
 354
 355     if( p_dec->fmt_in.video.i_frame_rate_base &&
 356         p_dec->fmt_in.video.i_frame_rate )
 357     {
 358         date_Change( &p_sys->dts, p_dec->fmt_in.video.i_frame_rate * 2,
 359                                   p_dec->fmt_in.video.i_frame_rate_base );
 360     }
 361
 362     if( b_avc )
 363     {
 364         /* This type of stream is produced by mp4 and matroska
 365          * when we want to store it in another streamformat, you need to convert
 366          * The fmt_in.p_extra should ALWAYS contain the avcC
 367          * The fmt_out.p_extra should contain all the SPS and PPS with 4 byte startcodes */
 368         if( h264_isavcC( p_dec->fmt_in.p_extra, p_dec->fmt_in.i_extra ) )
 369         {
 370             free( p_dec->fmt_out.p_extra );
 371             size_t i_size;
 372             p_dec->fmt_out.p_extra = h264_avcC_to_AnnexB_NAL( p_dec->fmt_in.p_extra,
 373                                                               p_dec->fmt_in.i_extra,
 374                                                              &i_size,
 375                                                              &p_sys->i_avcC_length_size );
 376             p_dec->fmt_out.i_extra = i_size;
 377             p_sys->b_recovered = !!p_dec->fmt_out.i_extra;
 378
 379             if(!p_dec->fmt_out.p_extra)
 380             {
 381                 msg_Err( p_dec, "Invalid AVC extradata");
 382                 Close( p_this );
 383                 return VLC_EGENERIC;
 384             }
 385         }
 386         else
 387         {
 388             msg_Err( p_dec, "Invalid or missing AVC extradata");
 389             Close( p_this );
 390             return VLC_EGENERIC;
 391         }
 392
 393         /* Set callback */
 394         p_dec->pf_packetize = PacketizeAVC1;
 395     }
 396     else
 397     {
 398         /* This type of stream contains data with 3 of 4 byte startcodes
 399          * The fmt_in.p_extra MAY contain SPS/PPS with 4 byte startcodes
 400          * The fmt_out.p_extra should be the same */
 401
 402         /* Set callback */
 403         p_dec->pf_packetize = Packetize;
 404     }
 405
 406     /* */
 407     if( p_dec->fmt_out.i_extra > 0 )
 408     {
 409         packetizer_Header( &p_sys->packetizer,
 410                            p_dec->fmt_out.p_extra, p_dec->fmt_out.i_extra );
 411     }
 412
 413     if( b_avc )
 414     {
 415         /* FIXME: that's not correct for every AVC */
 416         if( !p_sys->b_new_pps || !p_sys->b_new_sps )
 417         {
 418             msg_Err( p_dec, "Invalid or missing SPS %d or PPS %d in AVC extradata",
 419                      p_sys->b_new_sps, p_sys->b_new_pps );
 420             Close( p_this );
 421             return VLC_EGENERIC;
 422         }
 423
 424         msg_Dbg( p_dec, "Packetizer fed with AVC, nal length size=%d",
 425                          p_sys->i_avcC_length_size );
 426     }
 427
 428     /* CC are the same for H264/AVC in T35 sections (ETSI TS 101 154)  */
 429     p_dec->pf_get_cc = GetCc;
 430     p_dec->pf_flush = PacketizeFlush;
 431
 432     return VLC_SUCCESS;
 433 }
 434
 435 /*****************************************************************************
 436  * Close: clean up the packetizer
 437  *****************************************************************************/
 438 static void Close( vlc_object_t *p_this )
 439 {
 440     decoder_t *p_dec = (decoder_t*)p_this;
 441     decoder_sys_t *p_sys = p_dec->p_sys;
 442     int i;
 443
 444     DropStoredNAL( p_sys );
 445     for( i = 0; i <= H264_SPS_ID_MAX; i++ )
 446         StoreSPS( p_sys, i, NULL, NULL );
 447     for( i = 0; i <= H264_PPS_ID_MAX; i++ )
 448         StorePPS( p_sys, i, NULL, NULL );
 449
 450     packetizer_Clean( &p_sys->packetizer );
 451
 452     cc_storage_delete( p_sys->p_ccs );
 453
 454     free( p_sys );
 455 }
 456
 457 static void PacketizeFlush( decoder_t *p_dec )
 458 {
 459     decoder_sys_t *p_sys = p_dec->p_sys;
 460
 461     packetizer_Flush( &p_sys->packetizer );
 462 }
 463
 464 /****************************************************************************
 465  * Packetize: the whole thing
 466  * Search for the startcodes 3 or more bytes
 467  * Feed ParseNALBlock ALWAYS with 4 byte startcode prepended NALs
 468  ****************************************************************************/
 469 static block_t *Packetize( decoder_t *p_dec, block_t **pp_block )
 470 {
 471     decoder_sys_t *p_sys = p_dec->p_sys;
 472
 473     return packetizer_Packetize( &p_sys->packetizer, pp_block );
 474 }
 475
 476 /****************************************************************************
 477  * PacketizeAVC1: Takes VCL blocks of data and creates annexe B type NAL stream
 478  * Will always use 4 byte 0 0 0 1 startcodes
 479  * Will prepend a SPS and PPS before each keyframe
 480  ****************************************************************************/
 481 static block_t *PacketizeAVC1( decoder_t *p_dec, block_t **pp_block )
 482 {
 483     decoder_sys_t *p_sys = p_dec->p_sys;
 484
 485     return PacketizeXXC1( p_dec, p_sys->i_avcC_length_size,
 486                           pp_block, ParseNALBlock );
 487 }
 488
 489 /*****************************************************************************
 490  * GetCc:
 491  *****************************************************************************/
 492 static block_t *GetCc( decoder_t *p_dec, bool pb_present[4], int *pi_reorder_depth )
 493 {
 494     *pi_reorder_depth = 0;
 495     return cc_storage_get_current( p_dec->p_sys->p_ccs, pb_present );
 496 }
 497
 498 /****************************************************************************
 499  * Helpers
 500  ****************************************************************************/
 501 static void PacketizeReset( void *p_private, bool b_broken )
 502 {
 503     decoder_t *p_dec = p_private;
 504     decoder_sys_t *p_sys = p_dec->p_sys;
 505
 506     if( b_broken || !p_sys->b_slice )
 507     {
 508         DropStoredNAL( p_sys );
 509         p_sys->b_new_sps = false;
 510         p_sys->b_new_pps = false;
 511         p_sys->p_active_pps = NULL;
 512         p_sys->p_active_sps = NULL;
 513         p_sys->i_frame_pts = VLC_TS_INVALID;
 514         p_sys->i_frame_dts = VLC_TS_INVALID;
 515         p_sys->slice.type = H264_SLICE_TYPE_UNKNOWN;
 516         p_sys->b_slice = false;
 517         /* POC */
 518         h264_poc_context_init( &p_sys->pocctx );
 519         p_sys->prevdatedpoc.pts = VLC_TS_INVALID;
 520         /* From SEI */
 521         p_sys->i_dpb_output_delay = 0;
 522         p_sys->i_pic_struct = UINT8_MAX;
 523         p_sys->i_recovery_frame_cnt = UINT_MAX;
 524     }
 525     p_sys->i_next_block_flags = BLOCK_FLAG_DISCONTINUITY;
 526     p_sys->b_recovered = false;
 527     p_sys->i_recoveryfnum = UINT_MAX;
 528     date_Set( &p_sys->dts, VLC_TS_INVALID );
 529 }
 530 static block_t *PacketizeParse( void *p_private, bool *pb_ts_used, block_t *p_block )
 531 {
 532     decoder_t *p_dec = p_private;
 533
 534     /* Remove trailing 0 bytes */
 535     while( p_block->i_buffer > 5 && p_block->p_buffer[p_block->i_buffer-1] == 0x00 )
 536         p_block->i_buffer--;
 537
 538     return ParseNALBlock( p_dec, pb_ts_used, p_block );
 539 }
 540 static int PacketizeValidate( void *p_private, block_t *p_au )
 541 {
 542     VLC_UNUSED(p_private);
 543     VLC_UNUSED(p_au);
 544     return VLC_SUCCESS;
 545 }
 546
 547 /*****************************************************************************
 548  * ParseNALBlock: parses annexB type NALs
 549  * All p_frag blocks are required to start with 0 0 0 1 4-byte startcode
 550  *****************************************************************************/
 551 static block_t *ParseNALBlock( decoder_t *p_dec, bool *pb_ts_used, block_t *p_frag )
 552 {
 553     decoder_sys_t *p_sys = p_dec->p_sys;
 554     block_t *p_pic = NULL;
 555     bool b_new_picture = false;
 556
 557     const int i_nal_type = p_frag->p_buffer[4]&0x1f;
 558     const mtime_t i_frag_dts = p_frag->i_dts;
 559     const mtime_t i_frag_pts = p_frag->i_pts;
 560
 561     if( p_sys->b_slice && (!p_sys->p_active_pps || !p_sys->p_active_sps) )
 562     {
 563         msg_Warn( p_dec, "waiting for SPS/PPS" );
 564
 565         /* Reset context */
 566         p_sys->slice.type = H264_SLICE_TYPE_UNKNOWN;
 567         p_sys->b_slice = false;
 568         DropStoredNAL( p_sys );
 569         /* From SEI */
 570         p_sys->i_dpb_output_delay = 0;
 571         p_sys->i_pic_struct = UINT8_MAX;
 572         cc_storage_reset( p_sys->p_ccs );
 573     }
 574
 575     if( i_nal_type >= H264_NAL_SLICE && i_nal_type <= H264_NAL_SLICE_IDR )
 576     {
 577         h264_slice_t newslice;
 578
 579         if( i_nal_type == H264_NAL_SLICE_IDR )
 580         {
 581             p_sys->b_recovered = true;
 582             p_sys->i_recovery_frame_cnt = UINT_MAX;
 583             p_sys->i_recoveryfnum = UINT_MAX;
 584         }
 585
 586         if( ParseSliceHeader( p_dec, p_frag, &newslice ) )
 587         {
 588             /* Only IDR carries the id, to be propagated */
 589             if( newslice.i_idr_pic_id == -1 )
 590                 newslice.i_idr_pic_id = p_sys->slice.i_idr_pic_id;
 591
 592             b_new_picture = IsFirstVCLNALUnit( &p_sys->slice, &newslice );
 593             if( b_new_picture )
 594             {
 595                 /* Parse SEI for that frame now we should have matched SPS/PPS */
 596                 for( block_t *p_sei = p_sys->p_sei; p_sei; p_sei = p_sei->p_next )
 597                 {
 598                     HxxxParse_AnnexB_SEI( p_sei->p_buffer, p_sei->i_buffer,
 599                                           1 /* nal header */, ParseSeiCallback, p_dec );
 600                 }
 601
 602                 if( p_sys->b_slice )
 603                     p_pic = OutputPicture( p_dec );
 604             }
 605
 606             /* */
 607             p_sys->slice = newslice;
 608         }
 609         else
 610         {
 611             p_sys->p_active_pps = NULL;
 612             /* Fragment will be discarded later on */
 613         }
 614         p_sys->b_slice = true;
 615     }
 616     else if( i_nal_type == H264_NAL_SPS )
 617     {
 618         if( p_sys->b_slice )
 619             p_pic = OutputPicture( p_dec );
 620
 621         PutSPS( p_dec, p_frag );
 622         p_sys->b_new_sps = true;
 623
 624         /* Do not append the SPS because we will insert it on keyframes */
 625         p_frag = NULL;
 626     }
 627     else if( i_nal_type == H264_NAL_PPS )
 628     {
 629         if( p_sys->b_slice )
 630             p_pic = OutputPicture( p_dec );
 631
 632         PutPPS( p_dec, p_frag );
 633         p_sys->b_new_pps = true;
 634
 635         /* Do not append the PPS because we will insert it on keyframes */
 636         p_frag = NULL;
 637     }
 638     else if( i_nal_type == H264_NAL_SEI )
 639     {
 640         if( p_sys->b_slice )
 641             p_pic = OutputPicture( p_dec );
 642
 643         block_ChainLastAppend( &p_sys->pp_sei_last, p_frag );
 644         p_frag = NULL;
 645     }
 646     else if( i_nal_type == H264_NAL_END_OF_SEQ || i_nal_type == H264_NAL_END_OF_STREAM )
 647     {
 648         /* Early end of packetization */
 649         block_ChainLastAppend( &p_sys->pp_sei_last, p_frag );
 650         p_frag = NULL;
 651         /* important for still pictures/menus */
 652         p_sys->i_next_block_flags |= BLOCK_FLAG_END_OF_SEQUENCE;
 653         if( p_sys->b_slice )
 654             p_pic = OutputPicture( p_dec );
 655     }
 656     else if( i_nal_type == H264_NAL_AU_DELIMITER ||
 657              ( i_nal_type >= H264_NAL_PREFIX && i_nal_type <= H264_NAL_RESERVED_18 ) )
 658     {
 659         if( p_sys->b_slice )
 660             p_pic = OutputPicture( p_dec );
 661
 662         if( i_nal_type == H264_NAL_AU_DELIMITER )
 663         {
 664             if( p_sys->p_frame && (p_sys->p_frame->i_flags & BLOCK_FLAG_PRIVATE_AUD) )
 665             {
 666                 block_Release( p_frag );
 667                 p_frag = NULL;
 668             }
 669             else
 670             {
 671                 p_frag->i_flags |= BLOCK_FLAG_PRIVATE_AUD;
 672             }
 673         }
 674     }
 675
 676     /* Append the block */
 677     if( p_frag )
 678         block_ChainLastAppend( &p_sys->pp_frame_last, p_frag );
 679
 680     *pb_ts_used = false;
 681     if( p_sys->i_frame_dts <= VLC_TS_INVALID &&
 682         p_sys->i_frame_pts <= VLC_TS_INVALID && b_new_picture )
 683     {
 684         p_sys->i_frame_dts = i_frag_dts;
 685         p_sys->i_frame_pts = i_frag_pts;
 686         *pb_ts_used = true;
 687         if( i_frag_dts > VLC_TS_INVALID )
 688             date_Set( &p_sys->dts, i_frag_dts );
 689     }
 690
 691     if( p_pic && (p_pic->i_flags & BLOCK_FLAG_DROP) )
 692     {
 693         block_Release( p_pic );
 694         p_pic = NULL;
 695     }
 696
 697     return p_pic;
 698 }
 699
 700 static block_t *OutputPicture( decoder_t *p_dec )
 701 {
 702     decoder_sys_t *p_sys = p_dec->p_sys;
 703     block_t *p_pic = NULL;
 704     block_t **pp_pic_last = &p_pic;
 705
 706     if( unlikely(!p_sys->p_frame) )
 707     {
 708         assert( p_sys->p_frame );
 709         return NULL;
 710     }
 711
 712     /* Bind matched/referred PPS and SPS */
 713     const h264_picture_parameter_set_t *p_pps = p_sys->p_active_pps;
 714     const h264_sequence_parameter_set_t *p_sps = p_sys->p_active_sps;
 715     if( !p_pps || !p_sps )
 716     {
 717         DropStoredNAL( p_sys );
 718         return NULL;
 719     }
 720
 721     if( !p_sys->b_recovered && p_sys->i_recoveryfnum == UINT_MAX &&
 722          p_sys->i_recovery_frame_cnt == UINT_MAX && p_sys->slice.type == H264_SLICE_TYPE_I )
 723     {
 724         /* No way to recover using SEI, just sync on I Slice */
 725         p_sys->b_recovered = true;
 726     }
 727
 728     bool b_need_sps_pps = p_sys->slice.type == H264_SLICE_TYPE_I &&
 729                           p_sys->p_active_pps && p_sys->p_active_sps;
 730
 731     /* Handle SEI recovery */
 732     if ( !p_sys->b_recovered && p_sys->i_recovery_frame_cnt != UINT_MAX &&
 733          p_sys->i_recoveryfnum == UINT_MAX )
 734     {
 735         p_sys->i_recoveryfnum = p_sys->slice.i_frame_num + p_sys->i_recovery_frame_cnt;
 736         b_need_sps_pps = true; /* SPS/PPS must be inserted for SEI recovery */
 737         msg_Dbg( p_dec, "Recovering using SEI, prerolling %u reference pics", p_sys->i_recovery_frame_cnt );
 738     }
 739
 740     if( p_sys->i_recoveryfnum != UINT_MAX )
 741     {
 742         assert(p_sys->b_recovered == false);
 743         const unsigned maxFrameNum = 1 << (p_sps->i_log2_max_frame_num + 4);
 744         if( (p_sys->i_recoveryfnum > maxFrameNum &&
 745             (unsigned)p_sys->slice.i_frame_num <= maxFrameNum / 2 &&
 746             (unsigned)p_sys->slice.i_frame_num >= p_sys->i_recoveryfnum % maxFrameNum ) ||
 747             (unsigned)p_sys->slice.i_frame_num >= p_sys->i_recoveryfnum )
 748         {
 749             p_sys->i_recoveryfnum = UINT_MAX;
 750             p_sys->b_recovered = true;
 751             msg_Dbg( p_dec, "Recovery from SEI recovery point complete" );
 752         }
 753     }
 754
 755     /* Gather PPS/SPS if required */
 756     block_t *p_xpsnal = NULL;
 757     block_t **pp_xpsnal_tail = &p_xpsnal;
 758     if( b_need_sps_pps || p_sys->b_new_sps || p_sys->b_new_pps )
 759     {
 760         for( int i = 0; i <= H264_SPS_ID_MAX && (b_need_sps_pps || p_sys->b_new_sps); i++ )
 761         {
 762             if( p_sys->sps[i].p_block )
 763                 block_ChainLastAppend( &pp_xpsnal_tail, block_Duplicate( p_sys->sps[i].p_block ) );
 764         }
 765         for( int i = 0; i < H264_PPS_ID_MAX && (b_need_sps_pps || p_sys->b_new_pps); i++ )
 766         {
 767             if( p_sys->pps[i].p_block )
 768                 block_ChainLastAppend( &pp_xpsnal_tail, block_Duplicate( p_sys->pps[i].p_block ) );
 769         }
 770     }
 771
 772     /* Now rebuild NAL Sequence, inserting PPS/SPS if any */
 773     if( p_sys->p_frame->i_flags & BLOCK_FLAG_PRIVATE_AUD )
 774     {
 775         block_t *p_au = p_sys->p_frame;
 776         p_sys->p_frame = p_au->p_next;
 777         p_au->p_next = NULL;
 778         p_au->i_flags &= ~BLOCK_FLAG_PRIVATE_AUD;
 779         block_ChainLastAppend( &pp_pic_last, p_au );
 780     }
 781
 782     if( p_xpsnal )
 783         block_ChainLastAppend( &pp_pic_last, p_xpsnal );
 784
 785     if( p_sys->p_sei )
 786         block_ChainLastAppend( &pp_pic_last, p_sys->p_sei );
 787
 788     assert( p_sys->p_frame );
 789     if( p_sys->p_frame )
 790         block_ChainLastAppend( &pp_pic_last, p_sys->p_frame );
 791
 792     /* Reset chains, now empty */
 793     p_sys->p_frame = NULL;
 794     p_sys->pp_frame_last = &p_sys->p_frame;
 795     p_sys->p_sei = NULL;
 796     p_sys->pp_sei_last = &p_sys->p_sei;
 797
 798     p_pic = block_ChainGather( p_pic );
 799
 800     if( !p_pic )
 801         return NULL;
 802
 803     /* for PTS Fixup, interlaced fields (multiple AU/block) */
 804     int tFOC = 0, bFOC = 0, PictureOrderCount = 0;
 805     h264_compute_poc( p_sps, &p_sys->slice, &p_sys->pocctx, &PictureOrderCount, &tFOC, &bFOC );
 806
 807     unsigned i_num_clock_ts = h264_get_num_ts( p_sps, &p_sys->slice, p_sys->i_pic_struct, tFOC, bFOC );
 808
 809     if( p_sps->frame_mbs_only_flag == 0 && p_sps->vui.b_pic_struct_present_flag )
 810     {
 811         switch( p_sys->i_pic_struct )
 812         {
 813         /* Top and Bottom field slices */
 814         case 1:
 815         case 2:
 816             p_pic->i_flags |= BLOCK_FLAG_SINGLE_FIELD;
 817             p_pic->i_flags |= (!p_sys->slice.i_bottom_field_flag) ? BLOCK_FLAG_TOP_FIELD_FIRST
 818                                                                   : BLOCK_FLAG_BOTTOM_FIELD_FIRST;
 819             break;
 820         /* Each of the following slices contains multiple fields */
 821         case 3:
 822             p_pic->i_flags |= BLOCK_FLAG_TOP_FIELD_FIRST;
 823             break;
 824         case 4:
 825             p_pic->i_flags |= BLOCK_FLAG_BOTTOM_FIELD_FIRST;
 826             break;
 827         case 5:
 828             p_pic->i_flags |= BLOCK_FLAG_TOP_FIELD_FIRST;
 829             break;
 830         case 6:
 831             p_pic->i_flags |= BLOCK_FLAG_BOTTOM_FIELD_FIRST;
 832             break;
 833         default:
 834             break;
 835         }
 836     }
 837
 838     /* set dts/pts to current block timestamps */
 839     p_pic->i_dts = p_sys->i_frame_dts;
 840     p_pic->i_pts = p_sys->i_frame_pts;
 841
 842     /* Fixup missing timestamps after split (multiple AU/block)*/
 843     if( p_pic->i_dts <= VLC_TS_INVALID )
 844         p_pic->i_dts = date_Get( &p_sys->dts );
 845
 846     if( p_sys->slice.type == H264_SLICE_TYPE_I )
 847         p_sys->prevdatedpoc.pts = VLC_TS_INVALID;
 848
 849     if( p_pic->i_pts == VLC_TS_INVALID )
 850     {
 851         if( p_sys->prevdatedpoc.pts > VLC_TS_INVALID &&
 852             date_Get( &p_sys->dts ) != VLC_TS_INVALID )
 853         {
 854             date_t pts = p_sys->dts;
 855             date_Set( &pts, p_sys->prevdatedpoc.pts );
 856
 857             int diff = tFOC - p_sys->prevdatedpoc.num;
 858             if( diff > 0 )
 859                 date_Increment( &pts, diff );
 860             else
 861                 date_Decrement( &pts, -diff );
 862
 863             p_pic->i_pts = date_Get( &pts );
 864         }
 865         /* In case there's no PTS at all */
 866         else if( p_sys->slice.i_nal_ref_idc == 0 &&
 867                  p_sys->slice.type == H264_SLICE_TYPE_B )
 868         {
 869             p_pic->i_pts = p_pic->i_dts;
 870         }
 871         else if( p_sys->slice.type == H264_SLICE_TYPE_I &&
 872                  date_Get( &p_sys->dts ) != VLC_TS_INVALID )
 873         {
 874             /* Hell no PTS on IDR. We're totally blind */
 875             date_t pts = p_sys->dts;
 876             date_Increment( &pts, 2 );
 877             p_pic->i_pts = date_Get( &pts );
 878         }
 879     }
 880
 881     if( p_pic->i_pts > VLC_TS_INVALID )
 882     {
 883         p_sys->prevdatedpoc.pts = p_pic->i_pts;
 884         p_sys->prevdatedpoc.num = PictureOrderCount;
 885     }
 886
 887     if( p_pic->i_length == 0 )
 888     {
 889         if( p_sps->vui.i_time_scale )
 890         {
 891             p_pic->i_length = CLOCK_FREQ * i_num_clock_ts *
 892                               p_sps->vui.i_num_units_in_tick / p_sps->vui.i_time_scale;
 893         }
 894         else
 895         {
 896             date_t next = p_sys->dts;
 897             date_Increment( &next, i_num_clock_ts );
 898             p_pic->i_length = date_Get( &next ) - date_Get( &p_sys->dts );
 899         }
 900     }
 901
 902 #if 0
 903     msg_Err(p_dec, "F/BOC %d/%d POC %d %d rec %d flags %x ref%d fn %d fp %d %d pts %ld len %ld",
 904                     tFOC, bFOC, PictureOrderCount,
 905                     p_sys->slice.type, p_sys->b_recovered, p_pic->i_flags,
 906                     p_sys->slice.i_nal_ref_idc, p_sys->slice.i_frame_num, p_sys->slice.i_field_pic_flag,
 907                     p_pic->i_pts - p_pic->i_dts, p_pic->i_pts % (100*CLOCK_FREQ), p_pic->i_length);
 908 #endif
 909
 910     /* save for next pic fixups */
 911     if( date_Get( &p_sys->dts ) != VLC_TS_INVALID )
 912     {
 913         if( p_sys->i_next_block_flags & BLOCK_FLAG_DISCONTINUITY )
 914             date_Set( &p_sys->dts, VLC_TS_INVALID );
 915         else
 916             date_Increment( &p_sys->dts, i_num_clock_ts );
 917     }
 918
 919     if( p_pic )
 920     {
 921         p_pic->i_flags |= p_sys->i_next_block_flags;
 922         p_sys->i_next_block_flags = 0;
 923     }
 924
 925     switch( p_sys->slice.type )
 926     {
 927         case H264_SLICE_TYPE_P:
 928             p_pic->i_flags |= BLOCK_FLAG_TYPE_P;
 929             break;
 930         case H264_SLICE_TYPE_B:
 931             p_pic->i_flags |= BLOCK_FLAG_TYPE_B;
 932             break;
 933         case H264_SLICE_TYPE_I:
 934             p_pic->i_flags |= BLOCK_FLAG_TYPE_I;
 935         default:
 936             break;
 937     }
 938
 939     if( !p_sys->b_recovered )
 940     {
 941         if( p_sys->i_recoveryfnum != UINT_MAX ) /* recovering from SEI */
 942             p_pic->i_flags |= BLOCK_FLAG_PREROLL;
 943         else
 944             p_pic->i_flags |= BLOCK_FLAG_DROP;
 945     }
 946
 947     p_pic->i_flags &= ~BLOCK_FLAG_PRIVATE_AUD;
 948
 949     /* reset after output */
 950     p_sys->i_frame_dts = VLC_TS_INVALID;
 951     p_sys->i_frame_pts = VLC_TS_INVALID;
 952     p_sys->i_dpb_output_delay = 0;
 953     p_sys->i_pic_struct = UINT8_MAX;
 954     p_sys->i_recovery_frame_cnt = UINT_MAX;
 955     p_sys->slice.type = H264_SLICE_TYPE_UNKNOWN;
 956     p_sys->p_sei = NULL;
 957     p_sys->pp_sei_last = &p_sys->p_sei;
 958     p_sys->b_new_sps = false;
 959     p_sys->b_new_pps = false;
 960     p_sys->b_slice = false;
 961
 962     /* CC */
 963     cc_storage_commit( p_sys->p_ccs, p_pic );
 964
 965     return p_pic;
 966 }
 967
 968 static void PutSPS( decoder_t *p_dec, block_t *p_frag )
 969 {
 970     decoder_sys_t *p_sys = p_dec->p_sys;
 971
 972     const uint8_t *p_buffer = p_frag->p_buffer;
 973     size_t i_buffer = p_frag->i_buffer;
 974
 975     if( !hxxx_strip_AnnexB_startcode( &p_buffer, &i_buffer ) )
 976     {
 977         block_Release( p_frag );
 978         return;
 979     }
 980
 981     h264_sequence_parameter_set_t *p_sps = h264_decode_sps( p_buffer, i_buffer, true );
 982     if( !p_sps )
 983     {
 984         msg_Warn( p_dec, "invalid SPS" );
 985         block_Release( p_frag );
 986         return;
 987     }
 988
 989     /* We have a new SPS */
 990     if( !p_sys->sps[p_sps->i_id].p_sps )
 991         msg_Dbg( p_dec, "found NAL_SPS (sps_id=%d)", p_sps->i_id );
 992
 993     StoreSPS( p_sys, p_sps->i_id, p_frag, p_sps );
 994 }
 995
 996 static void PutPPS( decoder_t *p_dec, block_t *p_frag )
 997 {
 998     decoder_sys_t *p_sys = p_dec->p_sys;
 999     const uint8_t *p_buffer = p_frag->p_buffer;
1000     size_t i_buffer = p_frag->i_buffer;
1001
1002     if( !hxxx_strip_AnnexB_startcode( &p_buffer, &i_buffer ) )
1003     {
1004         block_Release( p_frag );
1005         return;
1006     }
1007
1008     h264_picture_parameter_set_t *p_pps = h264_decode_pps( p_buffer, i_buffer, true );
1009     if( !p_pps )
1010     {
1011         msg_Warn( p_dec, "invalid PPS" );
1012         block_Release( p_frag );
1013         return;
1014     }
1015
1016     /* We have a new PPS */
1017     if( !p_sys->pps[p_pps->i_id].p_pps )
1018         msg_Dbg( p_dec, "found NAL_PPS (pps_id=%d sps_id=%d)", p_pps->i_id, p_pps->i_sps_id );
1019
1020     StorePPS( p_sys, p_pps->i_id, p_frag, p_pps );
1021 }
1022
1023 static void GetSPSPPS( uint8_t i_pps_id, void *priv,
1024                        const h264_sequence_parameter_set_t **pp_sps,
1025                        const h264_picture_parameter_set_t **pp_pps )
1026 {
1027     decoder_sys_t *p_sys = priv;
1028
1029     *pp_pps = p_sys->pps[i_pps_id].p_pps;
1030     if( *pp_pps == NULL )
1031         *pp_sps = NULL;
1032     else
1033         *pp_sps = p_sys->sps[(*pp_pps)->i_sps_id].p_sps;
1034 }
1035
1036 static bool ParseSliceHeader( decoder_t *p_dec, const block_t *p_frag, h264_slice_t *p_slice )
1037 {
1038     decoder_sys_t *p_sys = p_dec->p_sys;
1039
1040     const uint8_t *p_stripped = p_frag->p_buffer;
1041     size_t i_stripped = p_frag->i_buffer;
1042
1043     if( !hxxx_strip_AnnexB_startcode( &p_stripped, &i_stripped ) || i_stripped < 2 )
1044         return false;
1045
1046     if( !h264_decode_slice( p_stripped, i_stripped, GetSPSPPS, p_sys, p_slice ) )
1047         return false;
1048
1049     const h264_sequence_parameter_set_t *p_sps;
1050     const h264_picture_parameter_set_t *p_pps;
1051     GetSPSPPS( p_slice->i_pic_parameter_set_id, p_sys, &p_sps, &p_pps );
1052     if( unlikely( !p_sps || !p_pps) )
1053         return false;
1054
1055     ActivateSets( p_dec, p_sps, p_pps );
1056
1057     return true;
1058 }
1059
1060 static bool ParseSeiCallback( const hxxx_sei_data_t *p_sei_data, void *cbdata )
1061 {
1062     decoder_t *p_dec = (decoder_t *) cbdata;
1063     decoder_sys_t *p_sys = p_dec->p_sys;
1064
1065     switch( p_sei_data->i_type )
1066     {
1067         /* Look for pic timing */
1068         case HXXX_SEI_PIC_TIMING:
1069         {
1070             const h264_sequence_parameter_set_t *p_sps = p_sys->p_active_sps;
1071             if( unlikely( p_sps == NULL ) )
1072             {
1073                 assert( p_sps );
1074                 break;
1075             }
1076
1077             if( p_sps->vui.b_valid )
1078             {
1079                 if( p_sps->vui.b_hrd_parameters_present_flag )
1080                 {
1081                     bs_read( p_sei_data->p_bs, p_sps->vui.i_cpb_removal_delay_length_minus1 + 1 );
1082                     p_sys->i_dpb_output_delay =
1083                             bs_read( p_sei_data->p_bs, p_sps->vui.i_dpb_output_delay_length_minus1 + 1 );
1084                 }
1085
1086                 if( p_sps->vui.b_pic_struct_present_flag )
1087                     p_sys->i_pic_struct = bs_read( p_sei_data->p_bs, 4 );
1088                 /* + unparsed remains */
1089             }
1090         } break;
1091
1092             /* Look for user_data_registered_itu_t_t35 */
1093         case HXXX_SEI_USER_DATA_REGISTERED_ITU_T_T35:
1094         {
1095             if( p_sei_data->itu_t35.type == HXXX_ITU_T35_TYPE_CC )
1096             {
1097                 cc_storage_append( p_sys->p_ccs, true, p_sei_data->itu_t35.u.cc.p_data,
1098                                                        p_sei_data->itu_t35.u.cc.i_data );
1099             }
1100         } break;
1101
1102             /* Look for SEI recovery point */
1103         case HXXX_SEI_RECOVERY_POINT:
1104         {
1105             if( !p_sys->b_recovered )
1106                 msg_Dbg( p_dec, "Seen SEI recovery point, %d recovery frames", p_sei_data->recovery.i_frames );
1107             p_sys->i_recovery_frame_cnt = p_sei_data->recovery.i_frames;
1108         } break;
1109
1110         default:
1111             /* Will skip */
1112             break;
1113     }
1114
1115     return true;
1116 }
1117