packetizer: h264: check leading nal
[vlc.git] / modules / packetizer / h264.c
blobfa66523c46aa301ce26b0f9aab914dd1b75c0fa7
1 /*****************************************************************************
2 * h264.c: h264/avc video packetizer
3 *****************************************************************************
4 * Copyright (C) 2001, 2002, 2006 VLC authors and VideoLAN
5 * $Id$
7 * Authors: Laurent Aimar <fenrir@via.ecp.fr>
8 * Eric Petit <titer@videolan.org>
9 * Gildas Bazin <gbazin@videolan.org>
10 * Derk-Jan Hartman <hartman at videolan dot org>
12 * This program is free software; you can redistribute it and/or modify it
13 * under the terms of the GNU Lesser General Public License as published by
14 * the Free Software Foundation; either version 2.1 of the License, or
15 * (at your option) any later version.
17 * This program is distributed in the hope that it will be useful,
18 * but WITHOUT ANY WARRANTY; without even the implied warranty of
19 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
20 * GNU Lesser General Public License for more details.
22 * You should have received a copy of the GNU Lesser General Public License
23 * along with this program; if not, write to the Free Software Foundation,
24 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
25 *****************************************************************************/
27 /*****************************************************************************
28 * Preamble
29 *****************************************************************************/
31 #ifdef HAVE_CONFIG_H
32 # include "config.h"
33 #endif
35 #include <vlc_common.h>
36 #include <vlc_plugin.h>
37 #include <vlc_sout.h>
38 #include <vlc_codec.h>
39 #include <vlc_block.h>
41 #include <vlc_block_helper.h>
42 #include <vlc_bits.h>
43 #include "h264_nal.h"
44 #include "h264_slice.h"
45 #include "hxxx_nal.h"
46 #include "hxxx_sei.h"
47 #include "hxxx_common.h"
48 #include "packetizer_helper.h"
49 #include "startcode_helper.h"
51 #include <limits.h>
53 /*****************************************************************************
54 * Module descriptor
55 *****************************************************************************/
56 static int Open ( vlc_object_t * );
57 static void Close( vlc_object_t * );
59 vlc_module_begin ()
60 set_category( CAT_SOUT )
61 set_subcategory( SUBCAT_SOUT_PACKETIZER )
62 set_description( N_("H.264 video packetizer") )
63 set_capability( "packetizer", 50 )
64 set_callbacks( Open, Close )
65 vlc_module_end ()
68 /****************************************************************************
69 * Local prototypes
70 ****************************************************************************/
72 typedef struct
74 /* */
75 packetizer_t packetizer;
77 /* */
78 bool b_slice;
79 struct
81 block_t *p_head;
82 block_t **pp_append;
83 } frame, leading;
85 /* a new sps/pps can be transmitted outside of iframes */
86 bool b_new_sps;
87 bool b_new_pps;
89 struct
91 block_t *p_block;
92 h264_sequence_parameter_set_t *p_sps;
93 } sps[H264_SPS_ID_MAX + 1];
94 struct
96 block_t *p_block;
97 h264_picture_parameter_set_t *p_pps;
98 } pps[H264_PPS_ID_MAX + 1];
99 const h264_sequence_parameter_set_t *p_active_sps;
100 const h264_picture_parameter_set_t *p_active_pps;
102 /* avcC data */
103 uint8_t i_avcC_length_size;
105 /* From SEI for current frame */
106 uint8_t i_pic_struct;
107 uint8_t i_dpb_output_delay;
108 unsigned i_recovery_frame_cnt;
110 /* Useful values of the Slice Header */
111 h264_slice_t slice;
113 /* */
114 int i_next_block_flags;
115 bool b_recovered;
116 unsigned i_recoveryfnum;
117 unsigned i_recoverystartfnum;
119 /* POC */
120 h264_poc_context_t pocctx;
121 struct
123 vlc_tick_t pts;
124 int num;
125 } prevdatedpoc;
127 vlc_tick_t i_frame_pts;
128 vlc_tick_t i_frame_dts;
130 date_t dts;
132 /* */
133 cc_storage_t *p_ccs;
134 } decoder_sys_t;
136 #define BLOCK_FLAG_PRIVATE_AUD (1 << BLOCK_FLAG_PRIVATE_SHIFT)
137 #define BLOCK_FLAG_PRIVATE_SEI (2 << BLOCK_FLAG_PRIVATE_SHIFT)
138 #define BLOCK_FLAG_DROP (4 << BLOCK_FLAG_PRIVATE_SHIFT)
140 static block_t *Packetize( decoder_t *, block_t ** );
141 static block_t *PacketizeAVC1( decoder_t *, block_t ** );
142 static block_t *GetCc( decoder_t *p_dec, decoder_cc_desc_t * );
143 static void PacketizeFlush( decoder_t * );
145 static void PacketizeReset( void *p_private, bool b_broken );
146 static block_t *PacketizeParse( void *p_private, bool *pb_ts_used, block_t * );
147 static int PacketizeValidate( void *p_private, block_t * );
149 static block_t *ParseNALBlock( decoder_t *, bool *pb_ts_used, block_t * );
151 static block_t *OutputPicture( decoder_t *p_dec );
152 static void PutSPS( decoder_t *p_dec, block_t *p_frag );
153 static void PutPPS( decoder_t *p_dec, block_t *p_frag );
154 static bool ParseSliceHeader( decoder_t *p_dec, const block_t *p_frag, h264_slice_t *p_slice );
155 static bool ParseSeiCallback( const hxxx_sei_data_t *, void * );
158 static const uint8_t p_h264_startcode[3] = { 0x00, 0x00, 0x01 };
160 /*****************************************************************************
161 * Helpers
162 *****************************************************************************/
164 static void StoreSPS( decoder_sys_t *p_sys, uint8_t i_id,
165 block_t *p_block, h264_sequence_parameter_set_t *p_sps )
167 if( p_sys->sps[i_id].p_block )
168 block_Release( p_sys->sps[i_id].p_block );
169 if( p_sys->sps[i_id].p_sps )
170 h264_release_sps( p_sys->sps[i_id].p_sps );
171 if( p_sys->sps[i_id].p_sps == p_sys->p_active_sps )
172 p_sys->p_active_sps = NULL;
173 p_sys->sps[i_id].p_block = p_block;
174 p_sys->sps[i_id].p_sps = p_sps;
177 static void StorePPS( decoder_sys_t *p_sys, uint8_t i_id,
178 block_t *p_block, h264_picture_parameter_set_t *p_pps )
180 if( p_sys->pps[i_id].p_block )
181 block_Release( p_sys->pps[i_id].p_block );
182 if( p_sys->pps[i_id].p_pps )
183 h264_release_pps( p_sys->pps[i_id].p_pps );
184 if( p_sys->pps[i_id].p_pps == p_sys->p_active_pps )
185 p_sys->p_active_pps = NULL;
186 p_sys->pps[i_id].p_block = p_block;
187 p_sys->pps[i_id].p_pps = p_pps;
190 static void ActivateSets( decoder_t *p_dec, const h264_sequence_parameter_set_t *p_sps,
191 const h264_picture_parameter_set_t *p_pps )
193 decoder_sys_t *p_sys = p_dec->p_sys;
195 p_sys->p_active_pps = p_pps;
196 p_sys->p_active_sps = p_sps;
198 if( p_sps )
200 p_dec->fmt_out.i_profile = p_sps->i_profile;
201 p_dec->fmt_out.i_level = p_sps->i_level;
203 (void) h264_get_picture_size( p_sps, &p_dec->fmt_out.video.i_width,
204 &p_dec->fmt_out.video.i_height,
205 &p_dec->fmt_out.video.i_visible_width,
206 &p_dec->fmt_out.video.i_visible_height );
208 if( p_sps->vui.i_sar_num != 0 && p_sps->vui.i_sar_den != 0 )
210 p_dec->fmt_out.video.i_sar_num = p_sps->vui.i_sar_num;
211 p_dec->fmt_out.video.i_sar_den = p_sps->vui.i_sar_den;
214 if( p_sps->vui.b_valid )
216 if( !p_dec->fmt_in.video.i_frame_rate_base &&
217 p_sps->vui.i_num_units_in_tick > 0 && p_sps->vui.i_time_scale > 1 )
219 const unsigned i_rate_base = p_sps->vui.i_num_units_in_tick;
220 const unsigned i_rate = p_sps->vui.i_time_scale >> 1; /* num_clock_ts == 2 */
221 if( i_rate_base != p_dec->fmt_out.video.i_frame_rate_base ||
222 i_rate != p_dec->fmt_out.video.i_frame_rate )
224 p_dec->fmt_out.video.i_frame_rate_base = i_rate_base;
225 p_dec->fmt_out.video.i_frame_rate = i_rate;
226 date_Change( &p_sys->dts, p_sps->vui.i_time_scale, p_sps->vui.i_num_units_in_tick );
229 if( p_dec->fmt_in.video.primaries == COLOR_PRIMARIES_UNDEF )
231 h264_get_colorimetry( p_sps, &p_dec->fmt_out.video.primaries,
232 &p_dec->fmt_out.video.transfer,
233 &p_dec->fmt_out.video.space,
234 &p_dec->fmt_out.video.color_range );
238 if( p_dec->fmt_out.i_extra == 0 && p_pps )
240 const block_t *p_spsblock = NULL;
241 const block_t *p_ppsblock = NULL;
242 for( size_t i=0; i<=H264_SPS_ID_MAX && !p_spsblock; i++ )
243 if( p_sps == p_sys->sps[i].p_sps )
244 p_spsblock = p_sys->sps[i].p_block;
246 for( size_t i=0; i<=H264_PPS_ID_MAX && !p_ppsblock; i++ )
247 if( p_pps == p_sys->pps[i].p_pps )
248 p_ppsblock = p_sys->pps[i].p_block;
250 if( p_spsblock && p_ppsblock )
252 size_t i_alloc = p_ppsblock->i_buffer + p_spsblock->i_buffer;
253 p_dec->fmt_out.p_extra = malloc( i_alloc );
254 if( p_dec->fmt_out.p_extra )
256 uint8_t*p_buf = p_dec->fmt_out.p_extra;
257 p_dec->fmt_out.i_extra = i_alloc;
258 memcpy( &p_buf[0], p_spsblock->p_buffer, p_spsblock->i_buffer );
259 memcpy( &p_buf[p_spsblock->i_buffer], p_ppsblock->p_buffer,
260 p_ppsblock->i_buffer );
267 static bool IsFirstVCLNALUnit( const h264_slice_t *p_prev, const h264_slice_t *p_cur )
269 /* Detection of the first VCL NAL unit of a primary coded picture
270 * (cf. 7.4.1.2.4) */
271 if( p_cur->i_frame_num != p_prev->i_frame_num ||
272 p_cur->i_pic_parameter_set_id != p_prev->i_pic_parameter_set_id ||
273 p_cur->i_field_pic_flag != p_prev->i_field_pic_flag ||
274 !p_cur->i_nal_ref_idc != !p_prev->i_nal_ref_idc )
275 return true;
276 if( (p_cur->i_bottom_field_flag != -1) &&
277 (p_prev->i_bottom_field_flag != -1) &&
278 (p_cur->i_bottom_field_flag != p_prev->i_bottom_field_flag) )
279 return true;
280 if( p_cur->i_pic_order_cnt_type == 0 &&
281 ( p_cur->i_pic_order_cnt_lsb != p_prev->i_pic_order_cnt_lsb ||
282 p_cur->i_delta_pic_order_cnt_bottom != p_prev->i_delta_pic_order_cnt_bottom ) )
283 return true;
284 else if( p_cur->i_pic_order_cnt_type == 1 &&
285 ( p_cur->i_delta_pic_order_cnt0 != p_prev->i_delta_pic_order_cnt0 ||
286 p_cur->i_delta_pic_order_cnt1 != p_prev->i_delta_pic_order_cnt1 ) )
287 return true;
288 if( ( p_cur->i_nal_type == H264_NAL_SLICE_IDR || p_prev->i_nal_type == H264_NAL_SLICE_IDR ) &&
289 ( p_cur->i_nal_type != p_prev->i_nal_type || p_cur->i_idr_pic_id != p_prev->i_idr_pic_id ) )
290 return true;
292 return false;
295 static void DropStoredNAL( decoder_sys_t *p_sys )
297 block_ChainRelease( p_sys->frame.p_head );
298 block_ChainRelease( p_sys->leading.p_head );
299 p_sys->frame.p_head = NULL;
300 p_sys->frame.pp_append = &p_sys->frame.p_head;
301 p_sys->leading.p_head = NULL;
302 p_sys->leading.pp_append = &p_sys->leading.p_head;
305 /*****************************************************************************
306 * Open: probe the packetizer and return score
307 * When opening after demux, the packetizer is only loaded AFTER the decoder
308 * That means that what you set in fmt_out is ignored by the decoder in this special case
309 *****************************************************************************/
310 static int Open( vlc_object_t *p_this )
312 decoder_t *p_dec = (decoder_t*)p_this;
313 decoder_sys_t *p_sys;
314 int i;
316 const bool b_avc = (p_dec->fmt_in.i_original_fourcc == VLC_FOURCC( 'a', 'v', 'c', '1' ));
318 if( p_dec->fmt_in.i_codec != VLC_CODEC_H264 )
319 return VLC_EGENERIC;
320 if( b_avc && p_dec->fmt_in.i_extra < 7 )
321 return VLC_EGENERIC;
323 /* Allocate the memory needed to store the decoder's structure */
324 if( ( p_dec->p_sys = p_sys = malloc( sizeof(decoder_sys_t) ) ) == NULL )
326 return VLC_ENOMEM;
329 p_sys->p_ccs = cc_storage_new();
330 if( unlikely(!p_sys->p_ccs) )
332 free( p_sys );
333 return VLC_ENOMEM;
336 packetizer_Init( &p_sys->packetizer,
337 p_h264_startcode, sizeof(p_h264_startcode), startcode_FindAnnexB,
338 p_h264_startcode, 1, 5,
339 PacketizeReset, PacketizeParse, PacketizeValidate, p_dec );
341 p_sys->b_slice = false;
342 p_sys->frame.p_head = NULL;
343 p_sys->frame.pp_append = &p_sys->frame.p_head;
344 p_sys->leading.p_head = NULL;
345 p_sys->leading.pp_append = &p_sys->leading.p_head;
346 p_sys->b_new_sps = false;
347 p_sys->b_new_pps = false;
349 for( i = 0; i <= H264_SPS_ID_MAX; i++ )
351 p_sys->sps[i].p_sps = NULL;
352 p_sys->sps[i].p_block = NULL;
354 p_sys->p_active_sps = NULL;
355 for( i = 0; i <= H264_PPS_ID_MAX; i++ )
357 p_sys->pps[i].p_pps = NULL;
358 p_sys->pps[i].p_block = NULL;
360 p_sys->p_active_pps = NULL;
361 p_sys->i_recovery_frame_cnt = UINT_MAX;
363 h264_slice_init( &p_sys->slice );
365 p_sys->i_next_block_flags = 0;
366 p_sys->b_recovered = false;
367 p_sys->i_recoveryfnum = UINT_MAX;
368 p_sys->i_frame_dts = VLC_TICK_INVALID;
369 p_sys->i_frame_pts = VLC_TICK_INVALID;
370 p_sys->i_dpb_output_delay = 0;
372 /* POC */
373 h264_poc_context_init( &p_sys->pocctx );
374 p_sys->prevdatedpoc.pts = VLC_TICK_INVALID;
376 date_Init( &p_sys->dts, 30000 * 2, 1001 );
378 /* Setup properties */
379 es_format_Copy( &p_dec->fmt_out, &p_dec->fmt_in );
380 p_dec->fmt_out.i_codec = VLC_CODEC_H264;
381 p_dec->fmt_out.b_packetized = true;
383 if( p_dec->fmt_in.video.i_frame_rate_base &&
384 p_dec->fmt_in.video.i_frame_rate &&
385 p_dec->fmt_in.video.i_frame_rate <= UINT_MAX / 2 )
387 date_Change( &p_sys->dts, p_dec->fmt_in.video.i_frame_rate * 2,
388 p_dec->fmt_in.video.i_frame_rate_base );
391 if( b_avc )
393 /* This type of stream is produced by mp4 and matroska
394 * when we want to store it in another streamformat, you need to convert
395 * The fmt_in.p_extra should ALWAYS contain the avcC
396 * The fmt_out.p_extra should contain all the SPS and PPS with 4 byte startcodes */
397 if( h264_isavcC( p_dec->fmt_in.p_extra, p_dec->fmt_in.i_extra ) )
399 free( p_dec->fmt_out.p_extra );
400 size_t i_size;
401 p_dec->fmt_out.p_extra = h264_avcC_to_AnnexB_NAL( p_dec->fmt_in.p_extra,
402 p_dec->fmt_in.i_extra,
403 &i_size,
404 &p_sys->i_avcC_length_size );
405 p_dec->fmt_out.i_extra = i_size;
406 p_sys->b_recovered = !!p_dec->fmt_out.i_extra;
408 if(!p_dec->fmt_out.p_extra)
410 msg_Err( p_dec, "Invalid AVC extradata");
411 Close( p_this );
412 return VLC_EGENERIC;
415 else
417 msg_Err( p_dec, "Invalid or missing AVC extradata");
418 Close( p_this );
419 return VLC_EGENERIC;
422 /* Set callback */
423 p_dec->pf_packetize = PacketizeAVC1;
425 else
427 /* This type of stream contains data with 3 of 4 byte startcodes
428 * The fmt_in.p_extra MAY contain SPS/PPS with 4 byte startcodes
429 * The fmt_out.p_extra should be the same */
431 /* Set callback */
432 p_dec->pf_packetize = Packetize;
435 /* */
436 if( p_dec->fmt_out.i_extra > 0 )
438 packetizer_Header( &p_sys->packetizer,
439 p_dec->fmt_out.p_extra, p_dec->fmt_out.i_extra );
442 if( b_avc )
444 /* FIXME: that's not correct for every AVC */
445 if( !p_sys->b_new_pps || !p_sys->b_new_sps )
447 msg_Err( p_dec, "Invalid or missing SPS %d or PPS %d in AVC extradata",
448 p_sys->b_new_sps, p_sys->b_new_pps );
449 Close( p_this );
450 return VLC_EGENERIC;
453 msg_Dbg( p_dec, "Packetizer fed with AVC, nal length size=%d",
454 p_sys->i_avcC_length_size );
457 /* CC are the same for H264/AVC in T35 sections (ETSI TS 101 154) */
458 p_dec->pf_get_cc = GetCc;
459 p_dec->pf_flush = PacketizeFlush;
461 return VLC_SUCCESS;
464 /*****************************************************************************
465 * Close: clean up the packetizer
466 *****************************************************************************/
467 static void Close( vlc_object_t *p_this )
469 decoder_t *p_dec = (decoder_t*)p_this;
470 decoder_sys_t *p_sys = p_dec->p_sys;
471 int i;
473 DropStoredNAL( p_sys );
474 for( i = 0; i <= H264_SPS_ID_MAX; i++ )
475 StoreSPS( p_sys, i, NULL, NULL );
476 for( i = 0; i <= H264_PPS_ID_MAX; i++ )
477 StorePPS( p_sys, i, NULL, NULL );
479 packetizer_Clean( &p_sys->packetizer );
481 cc_storage_delete( p_sys->p_ccs );
483 free( p_sys );
486 static void PacketizeFlush( decoder_t *p_dec )
488 decoder_sys_t *p_sys = p_dec->p_sys;
490 packetizer_Flush( &p_sys->packetizer );
493 /****************************************************************************
494 * Packetize: the whole thing
495 * Search for the startcodes 3 or more bytes
496 * Feed ParseNALBlock ALWAYS with 4 byte startcode prepended NALs
497 ****************************************************************************/
498 static block_t *Packetize( decoder_t *p_dec, block_t **pp_block )
500 decoder_sys_t *p_sys = p_dec->p_sys;
502 return packetizer_Packetize( &p_sys->packetizer, pp_block );
505 /****************************************************************************
506 * PacketizeAVC1: Takes VCL blocks of data and creates annexe B type NAL stream
507 * Will always use 4 byte 0 0 0 1 startcodes
508 * Will prepend a SPS and PPS before each keyframe
509 ****************************************************************************/
510 static block_t *PacketizeAVC1( decoder_t *p_dec, block_t **pp_block )
512 decoder_sys_t *p_sys = p_dec->p_sys;
514 return PacketizeXXC1( p_dec, p_sys->i_avcC_length_size,
515 pp_block, ParseNALBlock );
518 /*****************************************************************************
519 * GetCc:
520 *****************************************************************************/
521 static block_t *GetCc( decoder_t *p_dec, decoder_cc_desc_t *p_desc )
523 decoder_sys_t *p_sys = p_dec->p_sys;
524 return cc_storage_get_current( p_sys->p_ccs, p_desc );
527 /****************************************************************************
528 * Helpers
529 ****************************************************************************/
530 static void ResetOutputVariables( decoder_sys_t *p_sys )
532 p_sys->i_frame_dts = VLC_TICK_INVALID;
533 p_sys->i_frame_pts = VLC_TICK_INVALID;
534 p_sys->slice.type = H264_SLICE_TYPE_UNKNOWN;
535 p_sys->b_new_sps = false;
536 p_sys->b_new_pps = false;
537 p_sys->b_slice = false;
538 /* From SEI */
539 p_sys->i_dpb_output_delay = 0;
540 p_sys->i_pic_struct = UINT8_MAX;
541 p_sys->i_recovery_frame_cnt = UINT_MAX;
544 static void PacketizeReset( void *p_private, bool b_broken )
546 decoder_t *p_dec = p_private;
547 decoder_sys_t *p_sys = p_dec->p_sys;
549 if( b_broken || !p_sys->b_slice )
551 DropStoredNAL( p_sys );
552 ResetOutputVariables( p_sys );
553 p_sys->p_active_pps = NULL;
554 p_sys->p_active_sps = NULL;
555 /* POC */
556 h264_poc_context_init( &p_sys->pocctx );
557 p_sys->prevdatedpoc.pts = VLC_TICK_INVALID;
559 p_sys->i_next_block_flags = BLOCK_FLAG_DISCONTINUITY;
560 p_sys->b_recovered = false;
561 p_sys->i_recoveryfnum = UINT_MAX;
562 date_Set( &p_sys->dts, VLC_TICK_INVALID );
564 static block_t *PacketizeParse( void *p_private, bool *pb_ts_used, block_t *p_block )
566 decoder_t *p_dec = p_private;
568 /* Remove trailing 0 bytes */
569 while( p_block->i_buffer > 5 && p_block->p_buffer[p_block->i_buffer-1] == 0x00 )
570 p_block->i_buffer--;
572 return ParseNALBlock( p_dec, pb_ts_used, p_block );
574 static int PacketizeValidate( void *p_private, block_t *p_au )
576 VLC_UNUSED(p_private);
577 VLC_UNUSED(p_au);
578 return VLC_SUCCESS;
581 /*****************************************************************************
582 * ParseNALBlock: parses annexB type NALs
583 * All p_frag blocks are required to start with 0 0 0 1 4-byte startcode
584 *****************************************************************************/
585 static block_t *ParseNALBlock( decoder_t *p_dec, bool *pb_ts_used, block_t *p_frag )
587 decoder_sys_t *p_sys = p_dec->p_sys;
588 block_t *p_pic = NULL;
590 const int i_nal_type = p_frag->p_buffer[4]&0x1f;
591 const vlc_tick_t i_frag_dts = p_frag->i_dts;
592 const vlc_tick_t i_frag_pts = p_frag->i_pts;
594 if( p_sys->b_slice && (!p_sys->p_active_pps || !p_sys->p_active_sps) )
596 msg_Warn( p_dec, "waiting for SPS/PPS" );
598 /* Reset context */
599 DropStoredNAL( p_sys );
600 ResetOutputVariables( p_sys );
601 cc_storage_reset( p_sys->p_ccs );
604 switch( i_nal_type )
606 /*** Slices ***/
607 case H264_NAL_SLICE:
608 case H264_NAL_SLICE_DPA:
609 case H264_NAL_SLICE_DPB:
610 case H264_NAL_SLICE_DPC:
611 case H264_NAL_SLICE_IDR:
613 h264_slice_t newslice;
615 if( i_nal_type == H264_NAL_SLICE_IDR )
617 p_sys->b_recovered = true;
618 p_sys->i_recovery_frame_cnt = UINT_MAX;
619 p_sys->i_recoveryfnum = UINT_MAX;
622 if( ParseSliceHeader( p_dec, p_frag, &newslice ) )
624 /* Only IDR carries the id, to be propagated */
625 if( newslice.i_idr_pic_id == -1 )
626 newslice.i_idr_pic_id = p_sys->slice.i_idr_pic_id;
628 bool b_new_picture = IsFirstVCLNALUnit( &p_sys->slice, &newslice );
629 if( b_new_picture )
631 /* Parse SEI for that frame now we should have matched SPS/PPS */
632 for( block_t *p_sei = p_sys->leading.p_head; p_sei; p_sei = p_sei->p_next )
634 if( (p_sei->i_flags & BLOCK_FLAG_PRIVATE_SEI) == 0 )
635 continue;
636 HxxxParse_AnnexB_SEI( p_sei->p_buffer, p_sei->i_buffer,
637 1 /* nal header */, ParseSeiCallback, p_dec );
640 if( p_sys->b_slice )
641 p_pic = OutputPicture( p_dec );
644 /* */
645 p_sys->slice = newslice;
647 else
649 p_sys->p_active_pps = NULL;
650 /* Fragment will be discarded later on */
652 p_sys->b_slice = true;
654 block_ChainLastAppend( &p_sys->frame.pp_append, p_frag );
655 } break;
657 /*** Prefix NALs ***/
659 case H264_NAL_AU_DELIMITER:
660 if( p_sys->b_slice )
661 p_pic = OutputPicture( p_dec );
663 /* clear junk if no pic, we're always the first nal */
664 DropStoredNAL( p_sys );
666 p_frag->i_flags |= BLOCK_FLAG_PRIVATE_AUD;
668 block_ChainLastAppend( &p_sys->leading.pp_append, p_frag );
669 break;
671 case H264_NAL_SPS:
672 case H264_NAL_PPS:
673 if( p_sys->b_slice )
674 p_pic = OutputPicture( p_dec );
676 /* Stored for insert on keyframes */
677 if( i_nal_type == H264_NAL_SPS )
679 PutSPS( p_dec, p_frag );
680 p_sys->b_new_sps = true;
682 else
684 PutPPS( p_dec, p_frag );
685 p_sys->b_new_pps = true;
687 break;
689 case H264_NAL_SEI:
690 if( p_sys->b_slice )
691 p_pic = OutputPicture( p_dec );
693 p_frag->i_flags |= BLOCK_FLAG_PRIVATE_SEI;
694 block_ChainLastAppend( &p_sys->leading.pp_append, p_frag );
695 break;
697 case H264_NAL_SPS_EXT:
698 case H264_NAL_PREFIX: /* first slice/VCL associated data */
699 case H264_NAL_SUBSET_SPS:
700 case H264_NAL_DEPTH_PS:
701 case H264_NAL_RESERVED_17:
702 case H264_NAL_RESERVED_18:
703 if( p_sys->b_slice )
704 p_pic = OutputPicture( p_dec );
706 block_ChainLastAppend( &p_sys->leading.pp_append, p_frag );
707 break;
709 /*** Suffix NALs ***/
711 case H264_NAL_END_OF_SEQ:
712 case H264_NAL_END_OF_STREAM:
713 /* Early end of packetization */
714 block_ChainLastAppend( &p_sys->frame.pp_append, p_frag );
716 /* important for still pictures/menus */
717 p_sys->i_next_block_flags |= BLOCK_FLAG_END_OF_SEQUENCE;
718 if( p_sys->b_slice )
719 p_pic = OutputPicture( p_dec );
720 break;
722 case H264_NAL_SLICE_WP: // post
723 case H264_NAL_UNKNOWN:
724 case H264_NAL_FILLER_DATA:
725 case H264_NAL_SLICE_EXT:
726 case H264_NAL_SLICE_3D_EXT:
727 case H264_NAL_RESERVED_22:
728 case H264_NAL_RESERVED_23:
729 default: /* others 24..31, including unknown */
730 block_ChainLastAppend( &p_sys->frame.pp_append, p_frag );
731 break;
734 *pb_ts_used = false;
735 if( p_sys->i_frame_dts == VLC_TICK_INVALID &&
736 p_sys->i_frame_pts == VLC_TICK_INVALID )
738 p_sys->i_frame_dts = i_frag_dts;
739 p_sys->i_frame_pts = i_frag_pts;
740 *pb_ts_used = true;
741 if( i_frag_dts != VLC_TICK_INVALID )
742 date_Set( &p_sys->dts, i_frag_dts );
745 if( p_pic && (p_pic->i_flags & BLOCK_FLAG_DROP) )
747 block_Release( p_pic );
748 p_pic = NULL;
751 return p_pic;
754 static bool CanSwapPTSwithDTS( const h264_slice_t *p_slice,
755 const h264_sequence_parameter_set_t *p_sps )
757 if( p_slice->i_nal_ref_idc == 0 && p_slice->type == H264_SLICE_TYPE_B )
758 return true;
759 else if( p_sps->vui.b_valid )
760 return p_sps->vui.i_max_num_reorder_frames == 0;
761 else
762 return p_sps->i_profile == PROFILE_H264_CAVLC_INTRA;
765 static block_t *OutputPicture( decoder_t *p_dec )
767 decoder_sys_t *p_sys = p_dec->p_sys;
768 block_t *p_pic = NULL;
769 block_t **pp_pic_last = &p_pic;
771 if( unlikely(!p_sys->frame.p_head) )
773 assert( p_sys->frame.p_head );
774 DropStoredNAL( p_sys );
775 ResetOutputVariables( p_sys );
776 cc_storage_reset( p_sys->p_ccs );
777 return NULL;
780 /* Bind matched/referred PPS and SPS */
781 const h264_picture_parameter_set_t *p_pps = p_sys->p_active_pps;
782 const h264_sequence_parameter_set_t *p_sps = p_sys->p_active_sps;
783 if( !p_pps || !p_sps )
785 DropStoredNAL( p_sys );
786 ResetOutputVariables( p_sys );
787 cc_storage_reset( p_sys->p_ccs );
788 return NULL;
791 if( !p_sys->b_recovered && p_sys->i_recoveryfnum == UINT_MAX &&
792 p_sys->i_recovery_frame_cnt == UINT_MAX && p_sys->slice.type == H264_SLICE_TYPE_I )
794 /* No way to recover using SEI, just sync on I Slice */
795 p_sys->b_recovered = true;
798 bool b_need_sps_pps = p_sys->slice.type == H264_SLICE_TYPE_I &&
799 p_sys->p_active_pps && p_sys->p_active_sps;
801 /* Handle SEI recovery */
802 if ( !p_sys->b_recovered && p_sys->i_recovery_frame_cnt != UINT_MAX &&
803 p_sys->i_recoveryfnum == UINT_MAX )
805 p_sys->i_recoveryfnum = p_sys->slice.i_frame_num + p_sys->i_recovery_frame_cnt;
806 p_sys->i_recoverystartfnum = p_sys->slice.i_frame_num;
807 b_need_sps_pps = true; /* SPS/PPS must be inserted for SEI recovery */
808 msg_Dbg( p_dec, "Recovering using SEI, prerolling %u reference pics", p_sys->i_recovery_frame_cnt );
811 if( p_sys->i_recoveryfnum != UINT_MAX )
813 assert(p_sys->b_recovered == false);
814 const unsigned maxFrameNum = 1 << (p_sps->i_log2_max_frame_num + 4);
816 if( ( p_sys->i_recoveryfnum > maxFrameNum &&
817 p_sys->slice.i_frame_num < p_sys->i_recoverystartfnum &&
818 p_sys->slice.i_frame_num >= p_sys->i_recoveryfnum % maxFrameNum ) ||
819 ( p_sys->i_recoveryfnum <= maxFrameNum &&
820 p_sys->slice.i_frame_num >= p_sys->i_recoveryfnum ) )
822 p_sys->i_recoveryfnum = UINT_MAX;
823 p_sys->b_recovered = true;
824 msg_Dbg( p_dec, "Recovery from SEI recovery point complete" );
828 /* Gather PPS/SPS if required */
829 block_t *p_xpsnal = NULL;
830 block_t **pp_xpsnal_tail = &p_xpsnal;
831 if( b_need_sps_pps || p_sys->b_new_sps || p_sys->b_new_pps )
833 for( int i = 0; i <= H264_SPS_ID_MAX && (b_need_sps_pps || p_sys->b_new_sps); i++ )
835 if( p_sys->sps[i].p_block )
836 block_ChainLastAppend( &pp_xpsnal_tail, block_Duplicate( p_sys->sps[i].p_block ) );
838 for( int i = 0; i < H264_PPS_ID_MAX && (b_need_sps_pps || p_sys->b_new_pps); i++ )
840 if( p_sys->pps[i].p_block )
841 block_ChainLastAppend( &pp_xpsnal_tail, block_Duplicate( p_sys->pps[i].p_block ) );
845 /* Now rebuild NAL Sequence, inserting PPS/SPS if any */
846 if( p_sys->leading.p_head &&
847 (p_sys->leading.p_head->i_flags & BLOCK_FLAG_PRIVATE_AUD) )
849 block_t *p_au = p_sys->leading.p_head;
850 p_sys->leading.p_head = p_au->p_next;
851 p_au->p_next = NULL;
852 block_ChainLastAppend( &pp_pic_last, p_au );
855 if( p_xpsnal )
856 block_ChainLastAppend( &pp_pic_last, p_xpsnal );
858 if( p_sys->leading.p_head )
859 block_ChainLastAppend( &pp_pic_last, p_sys->leading.p_head );
861 assert( p_sys->frame.p_head );
862 if( p_sys->frame.p_head )
863 block_ChainLastAppend( &pp_pic_last, p_sys->frame.p_head );
865 /* Reset chains, now empty */
866 p_sys->frame.p_head = NULL;
867 p_sys->frame.pp_append = &p_sys->frame.p_head;
868 p_sys->leading.p_head = NULL;
869 p_sys->leading.pp_append = &p_sys->leading.p_head;
871 p_pic = block_ChainGather( p_pic );
873 if( !p_pic )
875 ResetOutputVariables( p_sys );
876 cc_storage_reset( p_sys->p_ccs );
877 return NULL;
880 /* clear up flags gathered */
881 p_pic->i_flags &= ~BLOCK_FLAG_PRIVATE_MASK;
883 /* for PTS Fixup, interlaced fields (multiple AU/block) */
884 int tFOC = 0, bFOC = 0, PictureOrderCount = 0;
885 h264_compute_poc( p_sps, &p_sys->slice, &p_sys->pocctx, &PictureOrderCount, &tFOC, &bFOC );
887 unsigned i_num_clock_ts = h264_get_num_ts( p_sps, &p_sys->slice, p_sys->i_pic_struct, tFOC, bFOC );
889 if( p_sps->frame_mbs_only_flag == 0 && p_sps->vui.b_pic_struct_present_flag )
891 switch( p_sys->i_pic_struct )
893 /* Top and Bottom field slices */
894 case 1:
895 case 2:
896 p_pic->i_flags |= BLOCK_FLAG_SINGLE_FIELD;
897 p_pic->i_flags |= (!p_sys->slice.i_bottom_field_flag) ? BLOCK_FLAG_TOP_FIELD_FIRST
898 : BLOCK_FLAG_BOTTOM_FIELD_FIRST;
899 break;
900 /* Each of the following slices contains multiple fields */
901 case 3:
902 p_pic->i_flags |= BLOCK_FLAG_TOP_FIELD_FIRST;
903 break;
904 case 4:
905 p_pic->i_flags |= BLOCK_FLAG_BOTTOM_FIELD_FIRST;
906 break;
907 case 5:
908 p_pic->i_flags |= BLOCK_FLAG_TOP_FIELD_FIRST;
909 break;
910 case 6:
911 p_pic->i_flags |= BLOCK_FLAG_BOTTOM_FIELD_FIRST;
912 break;
913 default:
914 break;
918 /* set dts/pts to current block timestamps */
919 p_pic->i_dts = p_sys->i_frame_dts;
920 p_pic->i_pts = p_sys->i_frame_pts;
922 /* Fixup missing timestamps after split (multiple AU/block)*/
923 if( p_pic->i_dts == VLC_TICK_INVALID )
924 p_pic->i_dts = date_Get( &p_sys->dts );
926 if( p_sys->slice.type == H264_SLICE_TYPE_I )
927 p_sys->prevdatedpoc.pts = VLC_TICK_INVALID;
929 if( p_pic->i_pts == VLC_TICK_INVALID )
931 if( p_sys->prevdatedpoc.pts != VLC_TICK_INVALID &&
932 date_Get( &p_sys->dts ) != VLC_TICK_INVALID )
934 date_t pts = p_sys->dts;
935 date_Set( &pts, p_sys->prevdatedpoc.pts );
937 int diff = tFOC - p_sys->prevdatedpoc.num;
938 if( diff > 0 )
939 date_Increment( &pts, diff );
940 else
941 date_Decrement( &pts, -diff );
943 p_pic->i_pts = date_Get( &pts );
945 /* In case there's no PTS at all */
946 else if( CanSwapPTSwithDTS( &p_sys->slice, p_sps ) )
948 p_pic->i_pts = p_pic->i_dts;
950 else if( p_sys->slice.type == H264_SLICE_TYPE_I &&
951 date_Get( &p_sys->dts ) != VLC_TICK_INVALID )
953 /* Hell no PTS on IDR. We're totally blind */
954 date_t pts = p_sys->dts;
955 date_Increment( &pts, 2 );
956 p_pic->i_pts = date_Get( &pts );
959 else if( p_pic->i_dts == VLC_TICK_INVALID &&
960 CanSwapPTSwithDTS( &p_sys->slice, p_sps ) )
962 p_pic->i_dts = p_pic->i_pts;
963 if( date_Get( &p_sys->dts ) == VLC_TICK_INVALID )
964 date_Set( &p_sys->dts, p_pic->i_pts );
967 if( p_pic->i_pts != VLC_TICK_INVALID )
969 p_sys->prevdatedpoc.pts = p_pic->i_pts;
970 p_sys->prevdatedpoc.num = PictureOrderCount;
973 if( p_pic->i_length == 0 )
975 if( p_sps->vui.i_time_scale )
977 p_pic->i_length = vlc_tick_from_samples(i_num_clock_ts *
978 p_sps->vui.i_num_units_in_tick, p_sps->vui.i_time_scale);
980 else
982 date_t next = p_sys->dts;
983 date_Increment( &next, i_num_clock_ts );
984 p_pic->i_length = date_Get( &next ) - date_Get( &p_sys->dts );
988 #if 0
989 msg_Err(p_dec, "F/BOC %d/%d POC %d %d rec %d flags %x ref%d fn %d fp %d %d pts %ld len %ld",
990 tFOC, bFOC, PictureOrderCount,
991 p_sys->slice.type, p_sys->b_recovered, p_pic->i_flags,
992 p_sys->slice.i_nal_ref_idc, p_sys->slice.i_frame_num, p_sys->slice.i_field_pic_flag,
993 p_pic->i_pts - p_pic->i_dts, p_pic->i_pts % VLC_TICK_FROM_SEC(100), p_pic->i_length);
994 #endif
996 /* save for next pic fixups */
997 if( date_Get( &p_sys->dts ) != VLC_TICK_INVALID )
999 if( p_sys->i_next_block_flags & BLOCK_FLAG_DISCONTINUITY )
1000 date_Set( &p_sys->dts, VLC_TICK_INVALID );
1001 else
1002 date_Increment( &p_sys->dts, i_num_clock_ts );
1005 if( p_pic )
1007 p_pic->i_flags |= p_sys->i_next_block_flags;
1008 p_sys->i_next_block_flags = 0;
1011 switch( p_sys->slice.type )
1013 case H264_SLICE_TYPE_P:
1014 p_pic->i_flags |= BLOCK_FLAG_TYPE_P;
1015 break;
1016 case H264_SLICE_TYPE_B:
1017 p_pic->i_flags |= BLOCK_FLAG_TYPE_B;
1018 break;
1019 case H264_SLICE_TYPE_I:
1020 p_pic->i_flags |= BLOCK_FLAG_TYPE_I;
1021 default:
1022 break;
1025 if( !p_sys->b_recovered )
1027 if( p_sys->i_recoveryfnum != UINT_MAX ) /* recovering from SEI */
1028 p_pic->i_flags |= BLOCK_FLAG_PREROLL;
1029 else
1030 p_pic->i_flags |= BLOCK_FLAG_DROP;
1033 p_pic->i_flags &= ~BLOCK_FLAG_PRIVATE_AUD;
1035 /* reset after output */
1036 ResetOutputVariables( p_sys );
1038 /* CC */
1039 cc_storage_commit( p_sys->p_ccs, p_pic );
1041 return p_pic;
1044 static void PutSPS( decoder_t *p_dec, block_t *p_frag )
1046 decoder_sys_t *p_sys = p_dec->p_sys;
1048 const uint8_t *p_buffer = p_frag->p_buffer;
1049 size_t i_buffer = p_frag->i_buffer;
1051 if( !hxxx_strip_AnnexB_startcode( &p_buffer, &i_buffer ) )
1053 block_Release( p_frag );
1054 return;
1057 h264_sequence_parameter_set_t *p_sps = h264_decode_sps( p_buffer, i_buffer, true );
1058 if( !p_sps )
1060 msg_Warn( p_dec, "invalid SPS" );
1061 block_Release( p_frag );
1062 return;
1065 /* We have a new SPS */
1066 if( !p_sys->sps[p_sps->i_id].p_sps )
1067 msg_Dbg( p_dec, "found NAL_SPS (sps_id=%d)", p_sps->i_id );
1069 StoreSPS( p_sys, p_sps->i_id, p_frag, p_sps );
1072 static void PutPPS( decoder_t *p_dec, block_t *p_frag )
1074 decoder_sys_t *p_sys = p_dec->p_sys;
1075 const uint8_t *p_buffer = p_frag->p_buffer;
1076 size_t i_buffer = p_frag->i_buffer;
1078 if( !hxxx_strip_AnnexB_startcode( &p_buffer, &i_buffer ) )
1080 block_Release( p_frag );
1081 return;
1084 h264_picture_parameter_set_t *p_pps = h264_decode_pps( p_buffer, i_buffer, true );
1085 if( !p_pps )
1087 msg_Warn( p_dec, "invalid PPS" );
1088 block_Release( p_frag );
1089 return;
1092 /* We have a new PPS */
1093 if( !p_sys->pps[p_pps->i_id].p_pps )
1094 msg_Dbg( p_dec, "found NAL_PPS (pps_id=%d sps_id=%d)", p_pps->i_id, p_pps->i_sps_id );
1096 StorePPS( p_sys, p_pps->i_id, p_frag, p_pps );
1099 static void GetSPSPPS( uint8_t i_pps_id, void *priv,
1100 const h264_sequence_parameter_set_t **pp_sps,
1101 const h264_picture_parameter_set_t **pp_pps )
1103 decoder_sys_t *p_sys = priv;
1105 *pp_pps = p_sys->pps[i_pps_id].p_pps;
1106 if( *pp_pps == NULL )
1107 *pp_sps = NULL;
1108 else
1109 *pp_sps = p_sys->sps[(*pp_pps)->i_sps_id].p_sps;
1112 static bool ParseSliceHeader( decoder_t *p_dec, const block_t *p_frag, h264_slice_t *p_slice )
1114 decoder_sys_t *p_sys = p_dec->p_sys;
1116 const uint8_t *p_stripped = p_frag->p_buffer;
1117 size_t i_stripped = p_frag->i_buffer;
1119 if( !hxxx_strip_AnnexB_startcode( &p_stripped, &i_stripped ) || i_stripped < 2 )
1120 return false;
1122 if( !h264_decode_slice( p_stripped, i_stripped, GetSPSPPS, p_sys, p_slice ) )
1123 return false;
1125 const h264_sequence_parameter_set_t *p_sps;
1126 const h264_picture_parameter_set_t *p_pps;
1127 GetSPSPPS( p_slice->i_pic_parameter_set_id, p_sys, &p_sps, &p_pps );
1128 if( unlikely( !p_sps || !p_pps) )
1129 return false;
1131 ActivateSets( p_dec, p_sps, p_pps );
1133 return true;
1136 static bool ParseSeiCallback( const hxxx_sei_data_t *p_sei_data, void *cbdata )
1138 decoder_t *p_dec = (decoder_t *) cbdata;
1139 decoder_sys_t *p_sys = p_dec->p_sys;
1141 switch( p_sei_data->i_type )
1143 /* Look for pic timing */
1144 case HXXX_SEI_PIC_TIMING:
1146 const h264_sequence_parameter_set_t *p_sps = p_sys->p_active_sps;
1147 if( unlikely( p_sps == NULL ) )
1149 assert( p_sps );
1150 break;
1153 if( p_sps->vui.b_valid )
1155 if( p_sps->vui.b_hrd_parameters_present_flag )
1157 bs_read( p_sei_data->p_bs, p_sps->vui.i_cpb_removal_delay_length_minus1 + 1 );
1158 p_sys->i_dpb_output_delay =
1159 bs_read( p_sei_data->p_bs, p_sps->vui.i_dpb_output_delay_length_minus1 + 1 );
1162 if( p_sps->vui.b_pic_struct_present_flag )
1163 p_sys->i_pic_struct = bs_read( p_sei_data->p_bs, 4 );
1164 /* + unparsed remains */
1166 } break;
1168 /* Look for user_data_registered_itu_t_t35 */
1169 case HXXX_SEI_USER_DATA_REGISTERED_ITU_T_T35:
1171 if( p_sei_data->itu_t35.type == HXXX_ITU_T35_TYPE_CC )
1173 cc_storage_append( p_sys->p_ccs, true, p_sei_data->itu_t35.u.cc.p_data,
1174 p_sei_data->itu_t35.u.cc.i_data );
1176 } break;
1178 case HXXX_SEI_FRAME_PACKING_ARRANGEMENT:
1180 if( p_dec->fmt_in.video.multiview_mode == MULTIVIEW_2D )
1182 video_multiview_mode_t mode;
1183 switch( p_sei_data->frame_packing.type )
1185 case FRAME_PACKING_INTERLEAVED_CHECKERBOARD:
1186 mode = MULTIVIEW_STEREO_CHECKERBOARD; break;
1187 case FRAME_PACKING_INTERLEAVED_COLUMN:
1188 mode = MULTIVIEW_STEREO_COL; break;
1189 case FRAME_PACKING_INTERLEAVED_ROW:
1190 mode = MULTIVIEW_STEREO_ROW; break;
1191 case FRAME_PACKING_SIDE_BY_SIDE:
1192 mode = MULTIVIEW_STEREO_SBS; break;
1193 case FRAME_PACKING_TOP_BOTTOM:
1194 mode = MULTIVIEW_STEREO_TB; break;
1195 case FRAME_PACKING_TEMPORAL:
1196 mode = MULTIVIEW_STEREO_FRAME; break;
1197 case FRAME_PACKING_TILED:
1198 default:
1199 mode = MULTIVIEW_2D; break;
1201 p_dec->fmt_out.video.multiview_mode = mode;
1203 } break;
1205 /* Look for SEI recovery point */
1206 case HXXX_SEI_RECOVERY_POINT:
1208 if( !p_sys->b_recovered )
1209 msg_Dbg( p_dec, "Seen SEI recovery point, %d recovery frames", p_sei_data->recovery.i_frames );
1210 p_sys->i_recovery_frame_cnt = p_sei_data->recovery.i_frames;
1211 } break;
1213 default:
1214 /* Will skip */
1215 break;
1218 return true;