1 /*****************************************************************************
2 * h264.c: h264/avc video packetizer
3 *****************************************************************************
4 * Copyright (C) 2001, 2002, 2006 VLC authors and VideoLAN
6 * Authors: Laurent Aimar <fenrir@via.ecp.fr>
7 * Eric Petit <titer@videolan.org>
8 * Gildas Bazin <gbazin@videolan.org>
9 * Derk-Jan Hartman <hartman at videolan dot org>
11 * This program is free software; you can redistribute it and/or modify it
12 * under the terms of the GNU Lesser General Public License as published by
13 * the Free Software Foundation; either version 2.1 of the License, or
14 * (at your option) any later version.
16 * This program is distributed in the hope that it will be useful,
17 * but WITHOUT ANY WARRANTY; without even the implied warranty of
18 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
19 * GNU Lesser General Public License for more details.
21 * You should have received a copy of the GNU Lesser General Public License
22 * along with this program; if not, write to the Free Software Foundation,
23 * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
24 *****************************************************************************/
26 /*****************************************************************************
28 *****************************************************************************/
34 #include <vlc_common.h>
35 #include <vlc_plugin.h>
37 #include <vlc_codec.h>
38 #include <vlc_block.h>
40 #include <vlc_block_helper.h>
43 #include "h264_slice.h"
46 #include "hxxx_common.h"
47 #include "packetizer_helper.h"
48 #include "startcode_helper.h"
52 /*****************************************************************************
54 *****************************************************************************/
55 static int Open ( vlc_object_t
* );
56 static void Close( vlc_object_t
* );
59 set_category( CAT_SOUT
)
60 set_subcategory( SUBCAT_SOUT_PACKETIZER
)
61 set_description( N_("H.264 video packetizer") )
62 set_capability( "packetizer", 50 )
63 set_callbacks( Open
, Close
)
67 /****************************************************************************
69 ****************************************************************************/
74 packetizer_t packetizer
;
84 /* a new sps/pps can be transmitted outside of iframes */
91 h264_sequence_parameter_set_t
*p_sps
;
92 } sps
[H264_SPS_ID_MAX
+ 1];
96 h264_picture_parameter_set_t
*p_pps
;
97 } pps
[H264_PPS_ID_MAX
+ 1];
98 const h264_sequence_parameter_set_t
*p_active_sps
;
99 const h264_picture_parameter_set_t
*p_active_pps
;
102 uint8_t i_avcC_length_size
;
104 /* From SEI for current frame */
105 uint8_t i_pic_struct
;
106 uint8_t i_dpb_output_delay
;
107 unsigned i_recovery_frame_cnt
;
109 /* Useful values of the Slice Header */
113 int i_next_block_flags
;
115 unsigned i_recoveryfnum
;
116 unsigned i_recoverystartfnum
;
119 h264_poc_context_t pocctx
;
126 vlc_tick_t i_frame_pts
;
127 vlc_tick_t i_frame_dts
;
135 #define BLOCK_FLAG_PRIVATE_AUD (1 << BLOCK_FLAG_PRIVATE_SHIFT)
136 #define BLOCK_FLAG_PRIVATE_SEI (2 << BLOCK_FLAG_PRIVATE_SHIFT)
137 #define BLOCK_FLAG_DROP (4 << BLOCK_FLAG_PRIVATE_SHIFT)
139 static block_t
*Packetize( decoder_t
*, block_t
** );
140 static block_t
*PacketizeAVC1( decoder_t
*, block_t
** );
141 static block_t
*GetCc( decoder_t
*p_dec
, decoder_cc_desc_t
* );
142 static void PacketizeFlush( decoder_t
* );
144 static void PacketizeReset( void *p_private
, bool b_broken
);
145 static block_t
*PacketizeParse( void *p_private
, bool *pb_ts_used
, block_t
* );
146 static int PacketizeValidate( void *p_private
, block_t
* );
147 static block_t
* PacketizeDrain( void *p_private
);
149 static block_t
*ParseNALBlock( decoder_t
*, bool *pb_ts_used
, block_t
* );
151 static block_t
*OutputPicture( decoder_t
*p_dec
);
152 static void PutSPS( decoder_t
*p_dec
, block_t
*p_frag
);
153 static void PutPPS( decoder_t
*p_dec
, block_t
*p_frag
);
154 static bool ParseSliceHeader( decoder_t
*p_dec
, const block_t
*p_frag
, h264_slice_t
*p_slice
);
155 static bool ParseSeiCallback( const hxxx_sei_data_t
*, void * );
158 static const uint8_t p_h264_startcode
[3] = { 0x00, 0x00, 0x01 };
160 /*****************************************************************************
162 *****************************************************************************/
164 static void StoreSPS( decoder_sys_t
*p_sys
, uint8_t i_id
,
165 block_t
*p_block
, h264_sequence_parameter_set_t
*p_sps
)
167 if( p_sys
->sps
[i_id
].p_block
)
168 block_Release( p_sys
->sps
[i_id
].p_block
);
169 if( p_sys
->sps
[i_id
].p_sps
)
170 h264_release_sps( p_sys
->sps
[i_id
].p_sps
);
171 if( p_sys
->sps
[i_id
].p_sps
== p_sys
->p_active_sps
)
172 p_sys
->p_active_sps
= NULL
;
173 p_sys
->sps
[i_id
].p_block
= p_block
;
174 p_sys
->sps
[i_id
].p_sps
= p_sps
;
177 static void StorePPS( decoder_sys_t
*p_sys
, uint8_t i_id
,
178 block_t
*p_block
, h264_picture_parameter_set_t
*p_pps
)
180 if( p_sys
->pps
[i_id
].p_block
)
181 block_Release( p_sys
->pps
[i_id
].p_block
);
182 if( p_sys
->pps
[i_id
].p_pps
)
183 h264_release_pps( p_sys
->pps
[i_id
].p_pps
);
184 if( p_sys
->pps
[i_id
].p_pps
== p_sys
->p_active_pps
)
185 p_sys
->p_active_pps
= NULL
;
186 p_sys
->pps
[i_id
].p_block
= p_block
;
187 p_sys
->pps
[i_id
].p_pps
= p_pps
;
190 static void ActivateSets( decoder_t
*p_dec
, const h264_sequence_parameter_set_t
*p_sps
,
191 const h264_picture_parameter_set_t
*p_pps
)
193 decoder_sys_t
*p_sys
= p_dec
->p_sys
;
195 p_sys
->p_active_pps
= p_pps
;
196 p_sys
->p_active_sps
= p_sps
;
200 p_dec
->fmt_out
.i_profile
= p_sps
->i_profile
;
201 p_dec
->fmt_out
.i_level
= p_sps
->i_level
;
203 (void) h264_get_picture_size( p_sps
, &p_dec
->fmt_out
.video
.i_width
,
204 &p_dec
->fmt_out
.video
.i_height
,
205 &p_dec
->fmt_out
.video
.i_visible_width
,
206 &p_dec
->fmt_out
.video
.i_visible_height
);
208 if( p_sps
->vui
.i_sar_num
!= 0 && p_sps
->vui
.i_sar_den
!= 0 )
210 p_dec
->fmt_out
.video
.i_sar_num
= p_sps
->vui
.i_sar_num
;
211 p_dec
->fmt_out
.video
.i_sar_den
= p_sps
->vui
.i_sar_den
;
214 if( !p_dec
->fmt_out
.video
.i_frame_rate
||
215 !p_dec
->fmt_out
.video
.i_frame_rate_base
)
217 /* on first run == if fmt_in does not provide frame rate info */
218 /* If we have frame rate info in the stream */
219 if(p_sps
->vui
.b_valid
&&
220 p_sps
->vui
.i_num_units_in_tick
> 0 &&
221 p_sps
->vui
.i_time_scale
> 1 )
223 date_Change( &p_sys
->dts
, p_sps
->vui
.i_time_scale
,
224 p_sps
->vui
.i_num_units_in_tick
);
226 /* else use the default num/den */
227 p_dec
->fmt_out
.video
.i_frame_rate
= p_sys
->dts
.i_divider_num
>> 1; /* num_clock_ts == 2 */
228 p_dec
->fmt_out
.video
.i_frame_rate_base
= p_sys
->dts
.i_divider_den
;
231 if( p_dec
->fmt_in
.video
.primaries
== COLOR_PRIMARIES_UNDEF
&&
234 h264_get_colorimetry( p_sps
, &p_dec
->fmt_out
.video
.primaries
,
235 &p_dec
->fmt_out
.video
.transfer
,
236 &p_dec
->fmt_out
.video
.space
,
237 &p_dec
->fmt_out
.video
.color_range
);
240 if( p_dec
->fmt_out
.i_extra
== 0 && p_pps
)
242 const block_t
*p_spsblock
= NULL
;
243 const block_t
*p_ppsblock
= NULL
;
244 for( size_t i
=0; i
<=H264_SPS_ID_MAX
&& !p_spsblock
; i
++ )
245 if( p_sps
== p_sys
->sps
[i
].p_sps
)
246 p_spsblock
= p_sys
->sps
[i
].p_block
;
248 for( size_t i
=0; i
<=H264_PPS_ID_MAX
&& !p_ppsblock
; i
++ )
249 if( p_pps
== p_sys
->pps
[i
].p_pps
)
250 p_ppsblock
= p_sys
->pps
[i
].p_block
;
252 if( p_spsblock
&& p_ppsblock
)
254 size_t i_alloc
= p_ppsblock
->i_buffer
+ p_spsblock
->i_buffer
;
255 p_dec
->fmt_out
.p_extra
= malloc( i_alloc
);
256 if( p_dec
->fmt_out
.p_extra
)
258 uint8_t*p_buf
= p_dec
->fmt_out
.p_extra
;
259 p_dec
->fmt_out
.i_extra
= i_alloc
;
260 memcpy( &p_buf
[0], p_spsblock
->p_buffer
, p_spsblock
->i_buffer
);
261 memcpy( &p_buf
[p_spsblock
->i_buffer
], p_ppsblock
->p_buffer
,
262 p_ppsblock
->i_buffer
);
269 static bool IsFirstVCLNALUnit( const h264_slice_t
*p_prev
, const h264_slice_t
*p_cur
)
271 /* Detection of the first VCL NAL unit of a primary coded picture
273 if( p_cur
->i_frame_num
!= p_prev
->i_frame_num
||
274 p_cur
->i_pic_parameter_set_id
!= p_prev
->i_pic_parameter_set_id
||
275 p_cur
->i_field_pic_flag
!= p_prev
->i_field_pic_flag
||
276 !p_cur
->i_nal_ref_idc
!= !p_prev
->i_nal_ref_idc
)
278 if( (p_cur
->i_bottom_field_flag
!= -1) &&
279 (p_prev
->i_bottom_field_flag
!= -1) &&
280 (p_cur
->i_bottom_field_flag
!= p_prev
->i_bottom_field_flag
) )
282 if( p_cur
->i_pic_order_cnt_type
== 0 &&
283 ( p_cur
->i_pic_order_cnt_lsb
!= p_prev
->i_pic_order_cnt_lsb
||
284 p_cur
->i_delta_pic_order_cnt_bottom
!= p_prev
->i_delta_pic_order_cnt_bottom
) )
286 else if( p_cur
->i_pic_order_cnt_type
== 1 &&
287 ( p_cur
->i_delta_pic_order_cnt0
!= p_prev
->i_delta_pic_order_cnt0
||
288 p_cur
->i_delta_pic_order_cnt1
!= p_prev
->i_delta_pic_order_cnt1
) )
290 if( ( p_cur
->i_nal_type
== H264_NAL_SLICE_IDR
|| p_prev
->i_nal_type
== H264_NAL_SLICE_IDR
) &&
291 ( p_cur
->i_nal_type
!= p_prev
->i_nal_type
|| p_cur
->i_idr_pic_id
!= p_prev
->i_idr_pic_id
) )
297 static void DropStoredNAL( decoder_sys_t
*p_sys
)
299 block_ChainRelease( p_sys
->frame
.p_head
);
300 block_ChainRelease( p_sys
->leading
.p_head
);
301 p_sys
->frame
.p_head
= NULL
;
302 p_sys
->frame
.pp_append
= &p_sys
->frame
.p_head
;
303 p_sys
->leading
.p_head
= NULL
;
304 p_sys
->leading
.pp_append
= &p_sys
->leading
.p_head
;
307 /*****************************************************************************
308 * Open: probe the packetizer and return score
309 * When opening after demux, the packetizer is only loaded AFTER the decoder
310 * That means that what you set in fmt_out is ignored by the decoder in this special case
311 *****************************************************************************/
312 static int Open( vlc_object_t
*p_this
)
314 decoder_t
*p_dec
= (decoder_t
*)p_this
;
315 decoder_sys_t
*p_sys
;
318 const bool b_avc
= (p_dec
->fmt_in
.i_original_fourcc
== VLC_FOURCC( 'a', 'v', 'c', '1' ));
320 if( p_dec
->fmt_in
.i_codec
!= VLC_CODEC_H264
)
322 if( b_avc
&& p_dec
->fmt_in
.i_extra
< 7 )
325 /* Allocate the memory needed to store the decoder's structure */
326 if( ( p_dec
->p_sys
= p_sys
= malloc( sizeof(decoder_sys_t
) ) ) == NULL
)
331 p_sys
->p_ccs
= cc_storage_new();
332 if( unlikely(!p_sys
->p_ccs
) )
338 packetizer_Init( &p_sys
->packetizer
,
339 p_h264_startcode
, sizeof(p_h264_startcode
), startcode_FindAnnexB
,
340 p_h264_startcode
, 1, 5,
341 PacketizeReset
, PacketizeParse
, PacketizeValidate
, PacketizeDrain
,
344 p_sys
->b_slice
= false;
345 p_sys
->frame
.p_head
= NULL
;
346 p_sys
->frame
.pp_append
= &p_sys
->frame
.p_head
;
347 p_sys
->leading
.p_head
= NULL
;
348 p_sys
->leading
.pp_append
= &p_sys
->leading
.p_head
;
349 p_sys
->b_new_sps
= false;
350 p_sys
->b_new_pps
= false;
352 for( i
= 0; i
<= H264_SPS_ID_MAX
; i
++ )
354 p_sys
->sps
[i
].p_sps
= NULL
;
355 p_sys
->sps
[i
].p_block
= NULL
;
357 p_sys
->p_active_sps
= NULL
;
358 for( i
= 0; i
<= H264_PPS_ID_MAX
; i
++ )
360 p_sys
->pps
[i
].p_pps
= NULL
;
361 p_sys
->pps
[i
].p_block
= NULL
;
363 p_sys
->p_active_pps
= NULL
;
364 p_sys
->i_recovery_frame_cnt
= UINT_MAX
;
366 h264_slice_init( &p_sys
->slice
);
368 p_sys
->i_next_block_flags
= 0;
369 p_sys
->b_recovered
= false;
370 p_sys
->i_recoveryfnum
= UINT_MAX
;
371 p_sys
->i_frame_dts
= VLC_TICK_INVALID
;
372 p_sys
->i_frame_pts
= VLC_TICK_INVALID
;
373 p_sys
->i_dpb_output_delay
= 0;
376 h264_poc_context_init( &p_sys
->pocctx
);
377 p_sys
->prevdatedpoc
.pts
= VLC_TICK_INVALID
;
379 date_Init( &p_sys
->dts
, 30000 * 2, 1001 );
381 /* Setup properties */
382 es_format_Copy( &p_dec
->fmt_out
, &p_dec
->fmt_in
);
383 p_dec
->fmt_out
.i_codec
= VLC_CODEC_H264
;
384 p_dec
->fmt_out
.b_packetized
= true;
386 if( p_dec
->fmt_in
.video
.i_frame_rate_base
&&
387 p_dec
->fmt_in
.video
.i_frame_rate
&&
388 p_dec
->fmt_in
.video
.i_frame_rate
<= UINT_MAX
/ 2 )
390 date_Change( &p_sys
->dts
, p_dec
->fmt_in
.video
.i_frame_rate
* 2,
391 p_dec
->fmt_in
.video
.i_frame_rate_base
);
396 /* This type of stream is produced by mp4 and matroska
397 * when we want to store it in another streamformat, you need to convert
398 * The fmt_in.p_extra should ALWAYS contain the avcC
399 * The fmt_out.p_extra should contain all the SPS and PPS with 4 byte startcodes */
400 if( h264_isavcC( p_dec
->fmt_in
.p_extra
, p_dec
->fmt_in
.i_extra
) )
402 free( p_dec
->fmt_out
.p_extra
);
404 p_dec
->fmt_out
.p_extra
= h264_avcC_to_AnnexB_NAL( p_dec
->fmt_in
.p_extra
,
405 p_dec
->fmt_in
.i_extra
,
407 &p_sys
->i_avcC_length_size
);
408 p_dec
->fmt_out
.i_extra
= i_size
;
409 p_sys
->b_recovered
= !!p_dec
->fmt_out
.i_extra
;
411 if(!p_dec
->fmt_out
.p_extra
)
413 msg_Err( p_dec
, "Invalid AVC extradata");
420 msg_Err( p_dec
, "Invalid or missing AVC extradata");
426 p_dec
->pf_packetize
= PacketizeAVC1
;
430 /* This type of stream contains data with 3 of 4 byte startcodes
431 * The fmt_in.p_extra MAY contain SPS/PPS with 4 byte startcodes
432 * The fmt_out.p_extra should be the same */
435 p_dec
->pf_packetize
= Packetize
;
439 if( p_dec
->fmt_out
.i_extra
> 0 )
441 packetizer_Header( &p_sys
->packetizer
,
442 p_dec
->fmt_out
.p_extra
, p_dec
->fmt_out
.i_extra
);
447 /* FIXME: that's not correct for every AVC */
448 if( !p_sys
->b_new_pps
|| !p_sys
->b_new_sps
)
450 msg_Err( p_dec
, "Invalid or missing SPS %d or PPS %d in AVC extradata",
451 p_sys
->b_new_sps
, p_sys
->b_new_pps
);
456 msg_Dbg( p_dec
, "Packetizer fed with AVC, nal length size=%d",
457 p_sys
->i_avcC_length_size
);
460 /* CC are the same for H264/AVC in T35 sections (ETSI TS 101 154) */
461 p_dec
->pf_get_cc
= GetCc
;
462 p_dec
->pf_flush
= PacketizeFlush
;
467 /*****************************************************************************
468 * Close: clean up the packetizer
469 *****************************************************************************/
470 static void Close( vlc_object_t
*p_this
)
472 decoder_t
*p_dec
= (decoder_t
*)p_this
;
473 decoder_sys_t
*p_sys
= p_dec
->p_sys
;
476 DropStoredNAL( p_sys
);
477 for( i
= 0; i
<= H264_SPS_ID_MAX
; i
++ )
478 StoreSPS( p_sys
, i
, NULL
, NULL
);
479 for( i
= 0; i
<= H264_PPS_ID_MAX
; i
++ )
480 StorePPS( p_sys
, i
, NULL
, NULL
);
482 packetizer_Clean( &p_sys
->packetizer
);
484 cc_storage_delete( p_sys
->p_ccs
);
489 static void PacketizeFlush( decoder_t
*p_dec
)
491 decoder_sys_t
*p_sys
= p_dec
->p_sys
;
493 packetizer_Flush( &p_sys
->packetizer
);
496 /****************************************************************************
497 * Packetize: the whole thing
498 * Search for the startcodes 3 or more bytes
499 * Feed ParseNALBlock ALWAYS with 4 byte startcode prepended NALs
500 ****************************************************************************/
501 static block_t
*Packetize( decoder_t
*p_dec
, block_t
**pp_block
)
503 decoder_sys_t
*p_sys
= p_dec
->p_sys
;
505 return packetizer_Packetize( &p_sys
->packetizer
, pp_block
);
508 /****************************************************************************
509 * PacketizeAVC1: Takes VCL blocks of data and creates annexe B type NAL stream
510 * Will always use 4 byte 0 0 0 1 startcodes
511 * Will prepend a SPS and PPS before each keyframe
512 ****************************************************************************/
513 static block_t
*PacketizeAVC1( decoder_t
*p_dec
, block_t
**pp_block
)
515 decoder_sys_t
*p_sys
= p_dec
->p_sys
;
517 return PacketizeXXC1( p_dec
, p_sys
->i_avcC_length_size
,
518 pp_block
, ParseNALBlock
);
521 /*****************************************************************************
523 *****************************************************************************/
524 static block_t
*GetCc( decoder_t
*p_dec
, decoder_cc_desc_t
*p_desc
)
526 decoder_sys_t
*p_sys
= p_dec
->p_sys
;
527 return cc_storage_get_current( p_sys
->p_ccs
, p_desc
);
530 /****************************************************************************
532 ****************************************************************************/
533 static void ResetOutputVariables( decoder_sys_t
*p_sys
)
535 p_sys
->i_frame_dts
= VLC_TICK_INVALID
;
536 p_sys
->i_frame_pts
= VLC_TICK_INVALID
;
537 p_sys
->slice
.type
= H264_SLICE_TYPE_UNKNOWN
;
538 p_sys
->b_new_sps
= false;
539 p_sys
->b_new_pps
= false;
540 p_sys
->b_slice
= false;
542 p_sys
->i_dpb_output_delay
= 0;
543 p_sys
->i_pic_struct
= UINT8_MAX
;
544 p_sys
->i_recovery_frame_cnt
= UINT_MAX
;
547 static void PacketizeReset( void *p_private
, bool b_flush
)
549 decoder_t
*p_dec
= p_private
;
550 decoder_sys_t
*p_sys
= p_dec
->p_sys
;
552 if( b_flush
|| !p_sys
->b_slice
)
554 DropStoredNAL( p_sys
);
555 ResetOutputVariables( p_sys
);
556 p_sys
->p_active_pps
= NULL
;
557 p_sys
->p_active_sps
= NULL
;
558 p_sys
->b_recovered
= false;
559 p_sys
->i_recoveryfnum
= UINT_MAX
;
561 h264_poc_context_init( &p_sys
->pocctx
);
562 p_sys
->prevdatedpoc
.pts
= VLC_TICK_INVALID
;
564 p_sys
->i_next_block_flags
= BLOCK_FLAG_DISCONTINUITY
;
565 date_Set( &p_sys
->dts
, VLC_TICK_INVALID
);
567 static block_t
*PacketizeParse( void *p_private
, bool *pb_ts_used
, block_t
*p_block
)
569 decoder_t
*p_dec
= p_private
;
571 /* Remove trailing 0 bytes */
572 while( p_block
->i_buffer
> 5 && p_block
->p_buffer
[p_block
->i_buffer
-1] == 0x00 )
575 return ParseNALBlock( p_dec
, pb_ts_used
, p_block
);
577 static int PacketizeValidate( void *p_private
, block_t
*p_au
)
579 VLC_UNUSED(p_private
);
584 static block_t
* PacketizeDrain( void *p_private
)
586 decoder_t
*p_dec
= p_private
;
587 decoder_sys_t
*p_sys
= p_dec
->p_sys
;
589 if( !p_sys
->b_slice
)
592 block_t
*p_out
= OutputPicture( p_dec
);
593 if( p_out
&& (p_out
->i_flags
& BLOCK_FLAG_DROP
) )
595 block_Release( p_out
);
602 /*****************************************************************************
603 * ParseNALBlock: parses annexB type NALs
604 * All p_frag blocks are required to start with 0 0 0 1 4-byte startcode
605 *****************************************************************************/
606 static block_t
*ParseNALBlock( decoder_t
*p_dec
, bool *pb_ts_used
, block_t
*p_frag
)
608 decoder_sys_t
*p_sys
= p_dec
->p_sys
;
609 block_t
*p_pic
= NULL
;
611 const int i_nal_type
= p_frag
->p_buffer
[4]&0x1f;
612 const vlc_tick_t i_frag_dts
= p_frag
->i_dts
;
613 const vlc_tick_t i_frag_pts
= p_frag
->i_pts
;
614 bool b_au_end
= p_frag
->i_flags
& BLOCK_FLAG_AU_END
;
615 p_frag
->i_flags
&= ~BLOCK_FLAG_AU_END
;
617 if( p_sys
->b_slice
&& (!p_sys
->p_active_pps
|| !p_sys
->p_active_sps
) )
619 msg_Warn( p_dec
, "waiting for SPS/PPS" );
622 DropStoredNAL( p_sys
);
623 ResetOutputVariables( p_sys
);
624 cc_storage_reset( p_sys
->p_ccs
);
631 case H264_NAL_SLICE_DPA
:
632 case H264_NAL_SLICE_DPB
:
633 case H264_NAL_SLICE_DPC
:
634 case H264_NAL_SLICE_IDR
:
636 h264_slice_t newslice
;
638 if( i_nal_type
== H264_NAL_SLICE_IDR
)
640 p_sys
->b_recovered
= true;
641 p_sys
->i_recovery_frame_cnt
= UINT_MAX
;
642 p_sys
->i_recoveryfnum
= UINT_MAX
;
645 if( ParseSliceHeader( p_dec
, p_frag
, &newslice
) )
647 /* Only IDR carries the id, to be propagated */
648 if( newslice
.i_idr_pic_id
== -1 )
649 newslice
.i_idr_pic_id
= p_sys
->slice
.i_idr_pic_id
;
651 bool b_new_picture
= IsFirstVCLNALUnit( &p_sys
->slice
, &newslice
);
654 /* Parse SEI for that frame now we should have matched SPS/PPS */
655 for( block_t
*p_sei
= p_sys
->leading
.p_head
; p_sei
; p_sei
= p_sei
->p_next
)
657 if( (p_sei
->i_flags
& BLOCK_FLAG_PRIVATE_SEI
) == 0 )
659 HxxxParse_AnnexB_SEI( p_sei
->p_buffer
, p_sei
->i_buffer
,
660 1 /* nal header */, ParseSeiCallback
, p_dec
);
664 p_pic
= OutputPicture( p_dec
);
668 p_sys
->slice
= newslice
;
672 p_sys
->p_active_pps
= NULL
;
673 /* Fragment will be discarded later on */
675 p_sys
->b_slice
= true;
677 block_ChainLastAppend( &p_sys
->frame
.pp_append
, p_frag
);
680 /*** Prefix NALs ***/
682 case H264_NAL_AU_DELIMITER
:
684 p_pic
= OutputPicture( p_dec
);
686 /* clear junk if no pic, we're always the first nal */
687 DropStoredNAL( p_sys
);
689 p_frag
->i_flags
|= BLOCK_FLAG_PRIVATE_AUD
;
691 block_ChainLastAppend( &p_sys
->leading
.pp_append
, p_frag
);
697 p_pic
= OutputPicture( p_dec
);
699 /* Stored for insert on keyframes */
700 if( i_nal_type
== H264_NAL_SPS
)
702 PutSPS( p_dec
, p_frag
);
703 p_sys
->b_new_sps
= true;
707 PutPPS( p_dec
, p_frag
);
708 p_sys
->b_new_pps
= true;
714 p_pic
= OutputPicture( p_dec
);
716 p_frag
->i_flags
|= BLOCK_FLAG_PRIVATE_SEI
;
717 block_ChainLastAppend( &p_sys
->leading
.pp_append
, p_frag
);
720 case H264_NAL_SPS_EXT
:
721 case H264_NAL_PREFIX
: /* first slice/VCL associated data */
722 case H264_NAL_SUBSET_SPS
:
723 case H264_NAL_DEPTH_PS
:
724 case H264_NAL_RESERVED_17
:
725 case H264_NAL_RESERVED_18
:
727 p_pic
= OutputPicture( p_dec
);
729 block_ChainLastAppend( &p_sys
->leading
.pp_append
, p_frag
);
732 /*** Suffix NALs ***/
734 case H264_NAL_END_OF_SEQ
:
735 case H264_NAL_END_OF_STREAM
:
736 /* Early end of packetization */
737 block_ChainLastAppend( &p_sys
->frame
.pp_append
, p_frag
);
739 /* important for still pictures/menus */
740 p_sys
->i_next_block_flags
|= BLOCK_FLAG_END_OF_SEQUENCE
;
742 p_pic
= OutputPicture( p_dec
);
745 case H264_NAL_SLICE_WP
: // post
746 case H264_NAL_UNKNOWN
:
747 case H264_NAL_FILLER_DATA
:
748 case H264_NAL_SLICE_EXT
:
749 case H264_NAL_SLICE_3D_EXT
:
750 case H264_NAL_RESERVED_22
:
751 case H264_NAL_RESERVED_23
:
752 default: /* others 24..31, including unknown */
753 block_ChainLastAppend( &p_sys
->frame
.pp_append
, p_frag
);
758 if( p_sys
->i_frame_dts
== VLC_TICK_INVALID
&&
759 p_sys
->i_frame_pts
== VLC_TICK_INVALID
)
761 p_sys
->i_frame_dts
= i_frag_dts
;
762 p_sys
->i_frame_pts
= i_frag_pts
;
764 if( i_frag_dts
!= VLC_TICK_INVALID
)
765 date_Set( &p_sys
->dts
, i_frag_dts
);
768 if( p_sys
->b_slice
&& b_au_end
&& !p_pic
)
770 p_pic
= OutputPicture( p_dec
);
773 if( p_pic
&& (p_pic
->i_flags
& BLOCK_FLAG_DROP
) )
775 block_Release( p_pic
);
782 static bool CanSwapPTSwithDTS( const h264_slice_t
*p_slice
,
783 const h264_sequence_parameter_set_t
*p_sps
)
785 if( p_slice
->i_nal_ref_idc
== 0 && p_slice
->type
== H264_SLICE_TYPE_B
)
787 else if( p_sps
->vui
.b_valid
)
788 return p_sps
->vui
.i_max_num_reorder_frames
== 0;
790 return p_sps
->i_profile
== PROFILE_H264_CAVLC_INTRA
;
793 static block_t
*OutputPicture( decoder_t
*p_dec
)
795 decoder_sys_t
*p_sys
= p_dec
->p_sys
;
796 block_t
*p_pic
= NULL
;
797 block_t
**pp_pic_last
= &p_pic
;
799 if( unlikely(!p_sys
->frame
.p_head
) )
801 assert( p_sys
->frame
.p_head
);
802 DropStoredNAL( p_sys
);
803 ResetOutputVariables( p_sys
);
804 cc_storage_reset( p_sys
->p_ccs
);
808 /* Bind matched/referred PPS and SPS */
809 const h264_picture_parameter_set_t
*p_pps
= p_sys
->p_active_pps
;
810 const h264_sequence_parameter_set_t
*p_sps
= p_sys
->p_active_sps
;
811 if( !p_pps
|| !p_sps
)
813 DropStoredNAL( p_sys
);
814 ResetOutputVariables( p_sys
);
815 cc_storage_reset( p_sys
->p_ccs
);
819 if( !p_sys
->b_recovered
&& p_sys
->i_recoveryfnum
== UINT_MAX
&&
820 p_sys
->i_recovery_frame_cnt
== UINT_MAX
&& p_sys
->slice
.type
== H264_SLICE_TYPE_I
)
822 /* No way to recover using SEI, just sync on I Slice */
823 p_sys
->b_recovered
= true;
826 bool b_need_sps_pps
= p_sys
->slice
.type
== H264_SLICE_TYPE_I
&&
827 p_sys
->p_active_pps
&& p_sys
->p_active_sps
;
829 /* Handle SEI recovery */
830 if ( !p_sys
->b_recovered
&& p_sys
->i_recovery_frame_cnt
!= UINT_MAX
&&
831 p_sys
->i_recoveryfnum
== UINT_MAX
)
833 p_sys
->i_recoveryfnum
= p_sys
->slice
.i_frame_num
+ p_sys
->i_recovery_frame_cnt
;
834 p_sys
->i_recoverystartfnum
= p_sys
->slice
.i_frame_num
;
835 b_need_sps_pps
= true; /* SPS/PPS must be inserted for SEI recovery */
836 msg_Dbg( p_dec
, "Recovering using SEI, prerolling %u reference pics", p_sys
->i_recovery_frame_cnt
);
839 if( p_sys
->i_recoveryfnum
!= UINT_MAX
)
841 assert(p_sys
->b_recovered
== false);
842 const unsigned maxFrameNum
= 1 << (p_sps
->i_log2_max_frame_num
+ 4);
844 if( ( p_sys
->i_recoveryfnum
> maxFrameNum
&&
845 p_sys
->slice
.i_frame_num
< p_sys
->i_recoverystartfnum
&&
846 p_sys
->slice
.i_frame_num
>= p_sys
->i_recoveryfnum
% maxFrameNum
) ||
847 ( p_sys
->i_recoveryfnum
<= maxFrameNum
&&
848 p_sys
->slice
.i_frame_num
>= p_sys
->i_recoveryfnum
) )
850 p_sys
->i_recoveryfnum
= UINT_MAX
;
851 p_sys
->b_recovered
= true;
852 msg_Dbg( p_dec
, "Recovery from SEI recovery point complete" );
856 /* Gather PPS/SPS if required */
857 block_t
*p_xpsnal
= NULL
;
858 block_t
**pp_xpsnal_tail
= &p_xpsnal
;
859 if( b_need_sps_pps
|| p_sys
->b_new_sps
|| p_sys
->b_new_pps
)
861 for( int i
= 0; i
<= H264_SPS_ID_MAX
&& (b_need_sps_pps
|| p_sys
->b_new_sps
); i
++ )
863 if( p_sys
->sps
[i
].p_block
)
864 block_ChainLastAppend( &pp_xpsnal_tail
, block_Duplicate( p_sys
->sps
[i
].p_block
) );
866 for( int i
= 0; i
< H264_PPS_ID_MAX
&& (b_need_sps_pps
|| p_sys
->b_new_pps
); i
++ )
868 if( p_sys
->pps
[i
].p_block
)
869 block_ChainLastAppend( &pp_xpsnal_tail
, block_Duplicate( p_sys
->pps
[i
].p_block
) );
873 /* Now rebuild NAL Sequence, inserting PPS/SPS if any */
874 if( p_sys
->leading
.p_head
&&
875 (p_sys
->leading
.p_head
->i_flags
& BLOCK_FLAG_PRIVATE_AUD
) )
877 block_t
*p_au
= p_sys
->leading
.p_head
;
878 p_sys
->leading
.p_head
= p_au
->p_next
;
880 block_ChainLastAppend( &pp_pic_last
, p_au
);
884 block_ChainLastAppend( &pp_pic_last
, p_xpsnal
);
886 if( p_sys
->leading
.p_head
)
887 block_ChainLastAppend( &pp_pic_last
, p_sys
->leading
.p_head
);
889 assert( p_sys
->frame
.p_head
);
890 if( p_sys
->frame
.p_head
)
891 block_ChainLastAppend( &pp_pic_last
, p_sys
->frame
.p_head
);
893 /* Reset chains, now empty */
894 p_sys
->frame
.p_head
= NULL
;
895 p_sys
->frame
.pp_append
= &p_sys
->frame
.p_head
;
896 p_sys
->leading
.p_head
= NULL
;
897 p_sys
->leading
.pp_append
= &p_sys
->leading
.p_head
;
899 p_pic
= block_ChainGather( p_pic
);
903 ResetOutputVariables( p_sys
);
904 cc_storage_reset( p_sys
->p_ccs
);
908 /* clear up flags gathered */
909 p_pic
->i_flags
&= ~BLOCK_FLAG_PRIVATE_MASK
;
911 /* for PTS Fixup, interlaced fields (multiple AU/block) */
912 int tFOC
= 0, bFOC
= 0, PictureOrderCount
= 0;
913 h264_compute_poc( p_sps
, &p_sys
->slice
, &p_sys
->pocctx
, &PictureOrderCount
, &tFOC
, &bFOC
);
915 unsigned i_num_clock_ts
= h264_get_num_ts( p_sps
, &p_sys
->slice
, p_sys
->i_pic_struct
, tFOC
, bFOC
);
917 if( p_sps
->frame_mbs_only_flag
== 0 && p_sps
->vui
.b_pic_struct_present_flag
)
919 switch( p_sys
->i_pic_struct
)
921 /* Top and Bottom field slices */
924 p_pic
->i_flags
|= BLOCK_FLAG_SINGLE_FIELD
;
925 p_pic
->i_flags
|= (!p_sys
->slice
.i_bottom_field_flag
) ? BLOCK_FLAG_TOP_FIELD_FIRST
926 : BLOCK_FLAG_BOTTOM_FIELD_FIRST
;
928 /* Each of the following slices contains multiple fields */
930 p_pic
->i_flags
|= BLOCK_FLAG_TOP_FIELD_FIRST
;
933 p_pic
->i_flags
|= BLOCK_FLAG_BOTTOM_FIELD_FIRST
;
936 p_pic
->i_flags
|= BLOCK_FLAG_TOP_FIELD_FIRST
;
939 p_pic
->i_flags
|= BLOCK_FLAG_BOTTOM_FIELD_FIRST
;
946 /* set dts/pts to current block timestamps */
947 p_pic
->i_dts
= p_sys
->i_frame_dts
;
948 p_pic
->i_pts
= p_sys
->i_frame_pts
;
950 /* Fixup missing timestamps after split (multiple AU/block)*/
951 if( p_pic
->i_dts
== VLC_TICK_INVALID
)
952 p_pic
->i_dts
= date_Get( &p_sys
->dts
);
954 if( p_sys
->slice
.type
== H264_SLICE_TYPE_I
)
955 p_sys
->prevdatedpoc
.pts
= VLC_TICK_INVALID
;
957 if( p_pic
->i_pts
== VLC_TICK_INVALID
)
959 if( p_sys
->prevdatedpoc
.pts
!= VLC_TICK_INVALID
&&
960 date_Get( &p_sys
->dts
) != VLC_TICK_INVALID
)
962 date_t pts
= p_sys
->dts
;
963 date_Set( &pts
, p_sys
->prevdatedpoc
.pts
);
965 int diff
= tFOC
- p_sys
->prevdatedpoc
.num
;
967 date_Increment( &pts
, diff
);
969 date_Decrement( &pts
, -diff
);
971 p_pic
->i_pts
= date_Get( &pts
);
972 /* non monotonically increasing dts on some videos 33333 33333...35000 */
973 if( p_pic
->i_pts
< p_pic
->i_dts
)
974 p_pic
->i_pts
= p_pic
->i_dts
;
976 /* In case there's no PTS at all */
977 else if( CanSwapPTSwithDTS( &p_sys
->slice
, p_sps
) )
979 p_pic
->i_pts
= p_pic
->i_dts
;
981 else if( p_sys
->slice
.type
== H264_SLICE_TYPE_I
&&
982 date_Get( &p_sys
->dts
) != VLC_TICK_INVALID
)
984 /* Hell no PTS on IDR. We're totally blind */
985 date_t pts
= p_sys
->dts
;
986 date_Increment( &pts
, 2 );
987 p_pic
->i_pts
= date_Get( &pts
);
990 else if( p_pic
->i_dts
== VLC_TICK_INVALID
&&
991 CanSwapPTSwithDTS( &p_sys
->slice
, p_sps
) )
993 p_pic
->i_dts
= p_pic
->i_pts
;
994 if( date_Get( &p_sys
->dts
) == VLC_TICK_INVALID
)
995 date_Set( &p_sys
->dts
, p_pic
->i_pts
);
998 if( p_pic
->i_pts
!= VLC_TICK_INVALID
)
1000 p_sys
->prevdatedpoc
.pts
= p_pic
->i_pts
;
1001 p_sys
->prevdatedpoc
.num
= PictureOrderCount
;
1004 if( p_pic
->i_length
== 0 )
1006 date_t next
= p_sys
->dts
;
1007 date_Increment( &next
, i_num_clock_ts
);
1008 p_pic
->i_length
= date_Get( &next
) - date_Get( &p_sys
->dts
);
1012 msg_Err(p_dec
, "F/BOC %d/%d POC %d %d rec %d flags %x ref%d fn %d fp %d %d pts %ld len %ld",
1013 tFOC
, bFOC
, PictureOrderCount
,
1014 p_sys
->slice
.type
, p_sys
->b_recovered
, p_pic
->i_flags
,
1015 p_sys
->slice
.i_nal_ref_idc
, p_sys
->slice
.i_frame_num
, p_sys
->slice
.i_field_pic_flag
,
1016 p_pic
->i_pts
- p_pic
->i_dts
, p_pic
->i_pts
% VLC_TICK_FROM_SEC(100), p_pic
->i_length
);
1019 /* save for next pic fixups */
1020 if( date_Get( &p_sys
->dts
) != VLC_TICK_INVALID
)
1022 if( p_sys
->i_next_block_flags
& BLOCK_FLAG_DISCONTINUITY
)
1023 date_Set( &p_sys
->dts
, VLC_TICK_INVALID
);
1025 date_Increment( &p_sys
->dts
, i_num_clock_ts
);
1030 p_pic
->i_flags
|= p_sys
->i_next_block_flags
;
1031 p_sys
->i_next_block_flags
= 0;
1034 switch( p_sys
->slice
.type
)
1036 case H264_SLICE_TYPE_P
:
1037 p_pic
->i_flags
|= BLOCK_FLAG_TYPE_P
;
1039 case H264_SLICE_TYPE_B
:
1040 p_pic
->i_flags
|= BLOCK_FLAG_TYPE_B
;
1042 case H264_SLICE_TYPE_I
:
1043 p_pic
->i_flags
|= BLOCK_FLAG_TYPE_I
;
1048 if( !p_sys
->b_recovered
)
1050 if( p_sys
->i_recoveryfnum
!= UINT_MAX
) /* recovering from SEI */
1051 p_pic
->i_flags
|= BLOCK_FLAG_PREROLL
;
1053 p_pic
->i_flags
|= BLOCK_FLAG_DROP
;
1056 p_pic
->i_flags
&= ~BLOCK_FLAG_PRIVATE_AUD
;
1058 /* reset after output */
1059 ResetOutputVariables( p_sys
);
1062 cc_storage_commit( p_sys
->p_ccs
, p_pic
);
1067 static void PutSPS( decoder_t
*p_dec
, block_t
*p_frag
)
1069 decoder_sys_t
*p_sys
= p_dec
->p_sys
;
1071 const uint8_t *p_buffer
= p_frag
->p_buffer
;
1072 size_t i_buffer
= p_frag
->i_buffer
;
1074 if( !hxxx_strip_AnnexB_startcode( &p_buffer
, &i_buffer
) )
1076 block_Release( p_frag
);
1080 h264_sequence_parameter_set_t
*p_sps
= h264_decode_sps( p_buffer
, i_buffer
, true );
1083 msg_Warn( p_dec
, "invalid SPS" );
1084 block_Release( p_frag
);
1088 /* We have a new SPS */
1089 if( !p_sys
->sps
[p_sps
->i_id
].p_sps
)
1090 msg_Dbg( p_dec
, "found NAL_SPS (sps_id=%d)", p_sps
->i_id
);
1092 StoreSPS( p_sys
, p_sps
->i_id
, p_frag
, p_sps
);
1095 static void PutPPS( decoder_t
*p_dec
, block_t
*p_frag
)
1097 decoder_sys_t
*p_sys
= p_dec
->p_sys
;
1098 const uint8_t *p_buffer
= p_frag
->p_buffer
;
1099 size_t i_buffer
= p_frag
->i_buffer
;
1101 if( !hxxx_strip_AnnexB_startcode( &p_buffer
, &i_buffer
) )
1103 block_Release( p_frag
);
1107 h264_picture_parameter_set_t
*p_pps
= h264_decode_pps( p_buffer
, i_buffer
, true );
1110 msg_Warn( p_dec
, "invalid PPS" );
1111 block_Release( p_frag
);
1115 /* We have a new PPS */
1116 if( !p_sys
->pps
[p_pps
->i_id
].p_pps
)
1117 msg_Dbg( p_dec
, "found NAL_PPS (pps_id=%d sps_id=%d)", p_pps
->i_id
, p_pps
->i_sps_id
);
1119 StorePPS( p_sys
, p_pps
->i_id
, p_frag
, p_pps
);
1122 static void GetSPSPPS( uint8_t i_pps_id
, void *priv
,
1123 const h264_sequence_parameter_set_t
**pp_sps
,
1124 const h264_picture_parameter_set_t
**pp_pps
)
1126 decoder_sys_t
*p_sys
= priv
;
1128 *pp_pps
= p_sys
->pps
[i_pps_id
].p_pps
;
1129 if( *pp_pps
== NULL
)
1132 *pp_sps
= p_sys
->sps
[(*pp_pps
)->i_sps_id
].p_sps
;
1135 static bool ParseSliceHeader( decoder_t
*p_dec
, const block_t
*p_frag
, h264_slice_t
*p_slice
)
1137 decoder_sys_t
*p_sys
= p_dec
->p_sys
;
1139 const uint8_t *p_stripped
= p_frag
->p_buffer
;
1140 size_t i_stripped
= p_frag
->i_buffer
;
1142 if( !hxxx_strip_AnnexB_startcode( &p_stripped
, &i_stripped
) || i_stripped
< 2 )
1145 if( !h264_decode_slice( p_stripped
, i_stripped
, GetSPSPPS
, p_sys
, p_slice
) )
1148 const h264_sequence_parameter_set_t
*p_sps
;
1149 const h264_picture_parameter_set_t
*p_pps
;
1150 GetSPSPPS( p_slice
->i_pic_parameter_set_id
, p_sys
, &p_sps
, &p_pps
);
1151 if( unlikely( !p_sps
|| !p_pps
) )
1154 ActivateSets( p_dec
, p_sps
, p_pps
);
1159 static bool ParseSeiCallback( const hxxx_sei_data_t
*p_sei_data
, void *cbdata
)
1161 decoder_t
*p_dec
= (decoder_t
*) cbdata
;
1162 decoder_sys_t
*p_sys
= p_dec
->p_sys
;
1164 switch( p_sei_data
->i_type
)
1166 /* Look for pic timing */
1167 case HXXX_SEI_PIC_TIMING
:
1169 const h264_sequence_parameter_set_t
*p_sps
= p_sys
->p_active_sps
;
1170 if( unlikely( p_sps
== NULL
) )
1176 if( p_sps
->vui
.b_valid
)
1178 if( p_sps
->vui
.b_hrd_parameters_present_flag
)
1180 bs_read( p_sei_data
->p_bs
, p_sps
->vui
.i_cpb_removal_delay_length_minus1
+ 1 );
1181 p_sys
->i_dpb_output_delay
=
1182 bs_read( p_sei_data
->p_bs
, p_sps
->vui
.i_dpb_output_delay_length_minus1
+ 1 );
1185 if( p_sps
->vui
.b_pic_struct_present_flag
)
1186 p_sys
->i_pic_struct
= bs_read( p_sei_data
->p_bs
, 4 );
1187 /* + unparsed remains */
1191 /* Look for user_data_registered_itu_t_t35 */
1192 case HXXX_SEI_USER_DATA_REGISTERED_ITU_T_T35
:
1194 if( p_sei_data
->itu_t35
.type
== HXXX_ITU_T35_TYPE_CC
)
1196 cc_storage_append( p_sys
->p_ccs
, true, p_sei_data
->itu_t35
.u
.cc
.p_data
,
1197 p_sei_data
->itu_t35
.u
.cc
.i_data
);
1201 case HXXX_SEI_FRAME_PACKING_ARRANGEMENT
:
1203 if( p_dec
->fmt_in
.video
.multiview_mode
== MULTIVIEW_2D
)
1205 video_multiview_mode_t mode
;
1206 switch( p_sei_data
->frame_packing
.type
)
1208 case FRAME_PACKING_INTERLEAVED_CHECKERBOARD
:
1209 mode
= MULTIVIEW_STEREO_CHECKERBOARD
; break;
1210 case FRAME_PACKING_INTERLEAVED_COLUMN
:
1211 mode
= MULTIVIEW_STEREO_COL
; break;
1212 case FRAME_PACKING_INTERLEAVED_ROW
:
1213 mode
= MULTIVIEW_STEREO_ROW
; break;
1214 case FRAME_PACKING_SIDE_BY_SIDE
:
1215 mode
= MULTIVIEW_STEREO_SBS
; break;
1216 case FRAME_PACKING_TOP_BOTTOM
:
1217 mode
= MULTIVIEW_STEREO_TB
; break;
1218 case FRAME_PACKING_TEMPORAL
:
1219 mode
= MULTIVIEW_STEREO_FRAME
; break;
1220 case FRAME_PACKING_TILED
:
1222 mode
= MULTIVIEW_2D
; break;
1224 p_dec
->fmt_out
.video
.multiview_mode
= mode
;
1228 /* Look for SEI recovery point */
1229 case HXXX_SEI_RECOVERY_POINT
:
1231 h264_sei_recovery_point_t reco
;
1232 if( !p_sys
->b_recovered
&&
1233 h264_decode_sei_recovery_point( p_sei_data
->p_bs
, &reco
) )
1235 msg_Dbg( p_dec
, "Seen SEI recovery point, %u recovery frames", reco
.i_frames
);
1236 p_sys
->i_recovery_frame_cnt
= reco
.i_frames
;