1 /*****************************************************************************
3 *****************************************************************************
4 * Copyright (C) 2011-2015 L-SMASH project
6 * Authors: Yusuke Nakamura <muken.the.vfrmaniac@gmail.com>
8 * Permission to use, copy, modify, and/or distribute this software for any
9 * purpose with or without fee is hereby granted, provided that the above
10 * copyright notice and this permission notice appear in all copies.
12 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
13 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
14 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
15 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
16 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
17 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
18 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
19 *****************************************************************************/
21 /* This file is available under an ISC license. */
23 #include "common/internal.h" /* must be placed first */
29 #define LSMASH_IMPORTER_INTERNAL
32 /***************************************************************************
34 ITU-T Recommendation H.264 (04/13)
36 ***************************************************************************/
37 #include "codecs/h264.h"
38 #include "codecs/nalu.h"
43 lsmash_entry_list_t avcC_list
[1]; /* stored as lsmash_codec_specific_t */
44 lsmash_media_ts_list_t ts_list
;
45 uint32_t max_au_length
;
46 uint32_t num_undecodable
;
49 uint64_t last_intra_cts
;
51 uint8_t composition_reordering_present
;
52 uint8_t field_pic_present
;
63 static void remove_h264_importer( h264_importer_t
*h264_imp
)
67 lsmash_remove_entries( h264_imp
->avcC_list
, lsmash_destroy_codec_specific_data
);
68 h264_cleanup_parser( &h264_imp
->info
);
69 lsmash_free( h264_imp
->ts_list
.timestamp
);
70 lsmash_free( h264_imp
);
73 static void h264_importer_cleanup( importer_t
*importer
)
75 debug_if( importer
&& importer
->info
)
76 remove_h264_importer( importer
->info
);
79 static h264_importer_t
*create_h264_importer( importer_t
*importer
)
81 h264_importer_t
*h264_imp
= lsmash_malloc_zero( sizeof(h264_importer_t
) );
84 if( h264_setup_parser( &h264_imp
->info
, 0 ) < 0 )
86 remove_h264_importer( h264_imp
);
89 lsmash_init_entry_list( h264_imp
->avcC_list
);
93 static inline int h264_complete_au( h264_access_unit_t
*au
, int probe
)
95 if( !au
->picture
.has_primary
|| au
->incomplete_length
== 0 )
98 memcpy( au
->data
, au
->incomplete_data
, au
->incomplete_length
);
99 au
->length
= au
->incomplete_length
;
100 au
->incomplete_length
= 0;
101 au
->picture
.has_primary
= 0;
105 static void h264_append_nalu_to_au( h264_access_unit_t
*au
, uint8_t *src_nalu
, uint32_t nalu_length
, int probe
)
109 uint8_t *dst_nalu
= au
->incomplete_data
+ au
->incomplete_length
+ NALU_DEFAULT_NALU_LENGTH_SIZE
;
110 for( int i
= NALU_DEFAULT_NALU_LENGTH_SIZE
; i
; i
-- )
111 *(dst_nalu
- i
) = (nalu_length
>> ((i
- 1) * 8)) & 0xff;
112 memcpy( dst_nalu
, src_nalu
, nalu_length
);
114 /* Note: au->incomplete_length shall be 0 immediately after AU has completed.
115 * Therefore, possible_au_length in h264_get_access_unit_internal() can't be used here
116 * to avoid increasing AU length monotonously through the entire stream. */
117 au
->incomplete_length
+= NALU_DEFAULT_NALU_LENGTH_SIZE
+ nalu_length
;
120 static int h264_get_au_internal_succeeded( h264_importer_t
*h264_imp
, h264_access_unit_t
*au
)
126 static int h264_get_au_internal_failed( h264_importer_t
*h264_imp
, h264_access_unit_t
*au
, int complete_au
, int ret
)
133 static lsmash_video_summary_t
*h264_create_summary
135 lsmash_h264_specific_parameters_t
*param
,
137 uint32_t max_au_length
140 lsmash_video_summary_t
*summary
= (lsmash_video_summary_t
*)lsmash_create_summary( LSMASH_SUMMARY_TYPE_VIDEO
);
143 /* Update summary here.
144 * max_au_length is set at the last of mp4sys_h264_probe function. */
145 lsmash_codec_specific_t
*cs
= lsmash_create_codec_specific_data( LSMASH_CODEC_SPECIFIC_DATA_TYPE_ISOM_VIDEO_H264
,
146 LSMASH_CODEC_SPECIFIC_FORMAT_UNSTRUCTURED
);
149 lsmash_cleanup_summary( (lsmash_summary_t
*)summary
);
152 cs
->data
.unstructured
= lsmash_create_h264_specific_info( param
, &cs
->size
);
153 if( !cs
->data
.unstructured
154 || lsmash_add_entry( &summary
->opaque
->list
, cs
) < 0 )
156 lsmash_cleanup_summary( (lsmash_summary_t
*)summary
);
157 lsmash_destroy_codec_specific_data( cs
);
160 summary
->sample_type
= ISOM_CODEC_TYPE_AVC1_VIDEO
;
161 summary
->max_au_length
= max_au_length
;
162 summary
->timescale
= sps
->vui
.time_scale
;
163 summary
->timebase
= sps
->vui
.num_units_in_tick
;
164 summary
->vfr
= !sps
->vui
.fixed_frame_rate_flag
;
165 summary
->sample_per_field
= 0;
166 summary
->width
= sps
->cropped_width
;
167 summary
->height
= sps
->cropped_height
;
168 summary
->par_h
= sps
->vui
.sar_width
;
169 summary
->par_v
= sps
->vui
.sar_height
;
170 summary
->color
.primaries_index
= sps
->vui
.colour_primaries
;
171 summary
->color
.transfer_index
= sps
->vui
.transfer_characteristics
;
172 summary
->color
.matrix_index
= sps
->vui
.matrix_coefficients
;
173 summary
->color
.full_range
= sps
->vui
.video_full_range_flag
;
177 static int h264_store_codec_specific
179 h264_importer_t
*h264_imp
,
180 lsmash_h264_specific_parameters_t
*avcC_param
183 lsmash_codec_specific_t
*src_cs
= lsmash_create_codec_specific_data( LSMASH_CODEC_SPECIFIC_DATA_TYPE_ISOM_VIDEO_H264
,
184 LSMASH_CODEC_SPECIFIC_FORMAT_STRUCTURED
);
186 return LSMASH_ERR_NAMELESS
;
187 lsmash_h264_specific_parameters_t
*src_param
= (lsmash_h264_specific_parameters_t
*)src_cs
->data
.structured
;
188 *src_param
= *avcC_param
;
189 lsmash_codec_specific_t
*dst_cs
= lsmash_convert_codec_specific_format( src_cs
, LSMASH_CODEC_SPECIFIC_FORMAT_STRUCTURED
);
190 src_param
->parameter_sets
= NULL
; /* Avoid freeing parameter sets within avcC_param. */
191 lsmash_destroy_codec_specific_data( src_cs
);
194 lsmash_destroy_codec_specific_data( dst_cs
);
195 return LSMASH_ERR_NAMELESS
;
197 if( lsmash_add_entry( h264_imp
->avcC_list
, dst_cs
) < 0 )
199 lsmash_destroy_codec_specific_data( dst_cs
);
200 return LSMASH_ERR_MEMORY_ALLOC
;
205 static inline void h264_new_access_unit
207 h264_access_unit_t
*au
211 au
->picture
.type
= H264_PICTURE_TYPE_NONE
;
212 au
->picture
.random_accessible
= 0;
213 au
->picture
.recovery_frame_cnt
= 0;
214 au
->picture
.has_mmco5
= 0;
215 au
->picture
.has_redundancy
= 0;
216 au
->picture
.broken_link_flag
= 0;
219 /* If probe equals 0, don't get the actual data (EBPS) of an access unit and only parse NALU.
220 * Currently, you can get AU of AVC video elemental stream only, not AVC parameter set elemental stream defined in 14496-15. */
221 static int h264_get_access_unit_internal
223 importer_t
*importer
,
227 h264_importer_t
*h264_imp
= (h264_importer_t
*)importer
->info
;
228 h264_info_t
*info
= &h264_imp
->info
;
229 h264_slice_info_t
*slice
= &info
->slice
;
230 h264_access_unit_t
*au
= &info
->au
;
231 h264_picture_info_t
*picture
= &au
->picture
;
232 h264_stream_buffer_t
*sb
= &info
->buffer
;
233 lsmash_bs_t
*bs
= importer
->bs
;
235 h264_new_access_unit( au
);
238 h264_nalu_header_t nuh
;
239 uint64_t start_code_length
;
240 uint64_t trailing_zero_bytes
;
241 uint64_t nalu_length
= h264_find_next_start_code( bs
, &nuh
, &start_code_length
, &trailing_zero_bytes
);
242 if( nalu_length
== NALU_NO_START_CODE_FOUND
)
244 /* For the last NALU.
245 * This NALU already has been appended into the latest access unit and parsed. */
246 h264_update_picture_info( info
, picture
, slice
, &info
->sei
);
247 complete_au
= h264_complete_au( au
, probe
);
249 return h264_get_au_internal_succeeded( h264_imp
, au
);
251 return h264_get_au_internal_failed( h264_imp
, au
, complete_au
, LSMASH_ERR_INVALID_DATA
);
253 uint8_t nalu_type
= nuh
.nal_unit_type
;
254 uint64_t next_sc_head_pos
= h264_imp
->sc_head_pos
257 + trailing_zero_bytes
;
261 fprintf( stderr
, "NALU type: %"PRIu8
" \n", nalu_type
);
262 fprintf( stderr
, " NALU header position: %"PRIx64
" \n", h264_imp
->sc_head_pos
+ start_code_length
);
263 fprintf( stderr
, " EBSP position: %"PRIx64
" \n", h264_imp
->sc_head_pos
+ start_code_length
+ nuh
.length
);
264 fprintf( stderr
, " EBSP length: %"PRIx64
" (%"PRIu64
") \n", nalu_length
- nuh
.length
, nalu_length
- nuh
.length
);
265 fprintf( stderr
, " trailing_zero_bytes: %"PRIx64
" \n", trailing_zero_bytes
);
266 fprintf( stderr
, " Next start code position: %"PRIx64
"\n", next_sc_head_pos
);
269 if( nalu_type
== H264_NALU_TYPE_FD
)
271 /* We don't support streams with both filler and HRD yet.
272 * Otherwise, just skip filler because 'avc1' and 'avc2' samples are forbidden to use filler. */
273 if( info
->sps
.vui
.hrd
.present
)
274 return h264_get_au_internal_failed( h264_imp
, au
, complete_au
, LSMASH_ERR_PATCH_WELCOME
);
276 else if( (nalu_type
>= H264_NALU_TYPE_SLICE_N_IDR
&& nalu_type
<= H264_NALU_TYPE_SPS_EXT
)
277 || nalu_type
== H264_NALU_TYPE_SLICE_AUX
)
280 /* Increase the buffer if needed. */
281 uint64_t possible_au_length
= au
->incomplete_length
+ NALU_DEFAULT_NALU_LENGTH_SIZE
+ nalu_length
;
282 if( sb
->bank
->buffer_size
< possible_au_length
283 && (err
= h264_supplement_buffer( sb
, au
, 2 * possible_au_length
)) < 0 )
285 lsmash_log( importer
, LSMASH_LOG_ERROR
, "failed to increase the buffer size.\n" );
286 return h264_get_au_internal_failed( h264_imp
, au
, complete_au
, err
);
288 /* Get the EBSP of the current NALU here.
289 * AVC elemental stream defined in 14496-15 can recognizes from 0 to 13, and 19 of nal_unit_type.
290 * We don't support SVC and MVC elemental stream defined in 14496-15 yet. */
291 uint8_t *nalu
= lsmash_bs_get_buffer_data( bs
) + start_code_length
;
292 if( nalu_type
>= H264_NALU_TYPE_SLICE_N_IDR
&& nalu_type
<= H264_NALU_TYPE_SLICE_IDR
)
294 /* VCL NALU (slice) */
295 h264_slice_info_t prev_slice
= *slice
;
296 if( (err
= h264_parse_slice( info
, &nuh
, sb
->rbsp
, nalu
+ nuh
.length
, nalu_length
- nuh
.length
)) < 0 )
297 return h264_get_au_internal_failed( h264_imp
, au
, complete_au
, err
);
298 if( probe
&& info
->avcC_pending
)
300 /* Copy and append a Codec Specific info. */
301 if( (err
= h264_store_codec_specific( h264_imp
, &info
->avcC_param
)) < 0 )
304 if( (err
= h264_move_pending_avcC_param( info
)) < 0 )
306 if( prev_slice
.present
)
308 /* Check whether the AU that contains the previous VCL NALU completed or not. */
309 if( h264_find_au_delimit_by_slice_info( slice
, &prev_slice
) )
311 /* The current NALU is the first VCL NALU of the primary coded picture of an new AU.
312 * Therefore, the previous slice belongs to the AU you want at this time. */
313 h264_update_picture_info( info
, picture
, &prev_slice
, &info
->sei
);
314 complete_au
= h264_complete_au( au
, probe
);
317 h264_update_picture_info_for_slice( info
, picture
, &prev_slice
);
319 h264_append_nalu_to_au( au
, nalu
, nalu_length
, probe
);
324 if( h264_find_au_delimit_by_nalu_type( nalu_type
, info
->prev_nalu_type
) )
326 /* The last slice belongs to the AU you want at this time. */
327 h264_update_picture_info( info
, picture
, slice
, &info
->sei
);
328 complete_au
= h264_complete_au( au
, probe
);
332 case H264_NALU_TYPE_SEI
:
334 if( (err
= h264_parse_sei( info
->bits
, &info
->sps
, &info
->sei
, sb
->rbsp
,
336 nalu_length
- nuh
.length
)) < 0 )
337 return h264_get_au_internal_failed( h264_imp
, au
, complete_au
, err
);
338 h264_append_nalu_to_au( au
, nalu
, nalu_length
, probe
);
341 case H264_NALU_TYPE_SPS
:
342 if( (err
= h264_try_to_append_parameter_set( info
, H264_PARAMETER_SET_TYPE_SPS
, nalu
, nalu_length
)) < 0 )
343 return h264_get_au_internal_failed( h264_imp
, au
, complete_au
, err
);
345 case H264_NALU_TYPE_PPS
:
346 if( (err
= h264_try_to_append_parameter_set( info
, H264_PARAMETER_SET_TYPE_PPS
, nalu
, nalu_length
)) < 0 )
347 return h264_get_au_internal_failed( h264_imp
, au
, complete_au
, err
);
349 case H264_NALU_TYPE_AUD
: /* We drop access unit delimiters. */
351 case H264_NALU_TYPE_SPS_EXT
:
352 if( (err
= h264_try_to_append_parameter_set( info
, H264_PARAMETER_SET_TYPE_SPSEXT
, nalu
, nalu_length
)) < 0 )
353 return h264_get_au_internal_failed( h264_imp
, au
, complete_au
, err
);
356 h264_append_nalu_to_au( au
, nalu
, nalu_length
, probe
);
359 if( info
->avcC_pending
)
360 importer
->status
= IMPORTER_CHANGE
;
363 /* Move to the first byte of the next start code. */
364 info
->prev_nalu_type
= nalu_type
;
365 if( lsmash_bs_read_seek( bs
, next_sc_head_pos
, SEEK_SET
) != next_sc_head_pos
)
367 lsmash_log( importer
, LSMASH_LOG_ERROR
, "failed to seek the next start code.\n" );
368 return h264_get_au_internal_failed( h264_imp
, au
, complete_au
, LSMASH_ERR_NAMELESS
);
370 /* Check if no more data to read from the stream. */
371 if( !lsmash_bs_is_end( bs
, NALU_SHORT_START_CODE_LENGTH
) )
372 h264_imp
->sc_head_pos
= next_sc_head_pos
;
373 /* If there is no more data in the stream, and flushed chunk of NALUs, flush it as complete AU here. */
374 else if( au
->incomplete_length
&& au
->length
== 0 )
376 h264_update_picture_info( info
, picture
, slice
, &info
->sei
);
377 h264_complete_au( au
, probe
);
378 return h264_get_au_internal_succeeded( h264_imp
, au
);
381 return h264_get_au_internal_succeeded( h264_imp
, au
);
385 static inline void h264_importer_check_eof( importer_t
*importer
, h264_access_unit_t
*au
)
387 /* AVC byte stream NALU consists of at least 4 bytes (start-code + NALU-header). */
388 if( lsmash_bs_is_end( importer
->bs
, NALU_SHORT_START_CODE_LENGTH
) && au
->incomplete_length
== 0 )
389 importer
->status
= IMPORTER_EOF
;
390 else if( importer
->status
!= IMPORTER_CHANGE
)
391 importer
->status
= IMPORTER_OK
;
394 static int h264_importer_get_accessunit
396 importer_t
*importer
,
397 uint32_t track_number
,
398 lsmash_sample_t
**p_sample
401 if( !importer
->info
)
402 return LSMASH_ERR_NAMELESS
;
403 if( track_number
!= 1 )
404 return LSMASH_ERR_FUNCTION_PARAM
;
405 h264_importer_t
*h264_imp
= (h264_importer_t
*)importer
->info
;
406 h264_info_t
*info
= &h264_imp
->info
;
407 importer_status current_status
= importer
->status
;
408 if( current_status
== IMPORTER_ERROR
)
409 return LSMASH_ERR_NAMELESS
;
410 if( current_status
== IMPORTER_EOF
)
412 int err
= h264_get_access_unit_internal( importer
, 0 );
415 importer
->status
= IMPORTER_ERROR
;
418 h264_importer_check_eof( importer
, &info
->au
);
419 if( importer
->status
== IMPORTER_CHANGE
&& !info
->avcC_pending
)
420 current_status
= IMPORTER_CHANGE
;
421 if( current_status
== IMPORTER_CHANGE
)
423 /* Update the active summary. */
424 lsmash_codec_specific_t
*cs
= (lsmash_codec_specific_t
*)lsmash_get_entry_data( h264_imp
->avcC_list
, ++ h264_imp
->avcC_number
);
426 return LSMASH_ERR_NAMELESS
;
427 lsmash_h264_specific_parameters_t
*avcC_param
= (lsmash_h264_specific_parameters_t
*)cs
->data
.structured
;
428 lsmash_video_summary_t
*summary
= h264_create_summary( avcC_param
, &info
->sps
, h264_imp
->max_au_length
);
430 return LSMASH_ERR_NAMELESS
;
431 lsmash_remove_entry( importer
->summaries
, track_number
, lsmash_cleanup_summary
);
432 if( lsmash_add_entry( importer
->summaries
, summary
) < 0 )
434 lsmash_cleanup_summary( (lsmash_summary_t
*)summary
);
435 return LSMASH_ERR_MEMORY_ALLOC
;
437 importer
->status
= IMPORTER_OK
;
439 lsmash_sample_t
*sample
= lsmash_create_sample( h264_imp
->max_au_length
);
441 return LSMASH_ERR_MEMORY_ALLOC
;
443 h264_access_unit_t
*au
= &info
->au
;
444 h264_picture_info_t
*picture
= &au
->picture
;
445 sample
->dts
= h264_imp
->ts_list
.timestamp
[ au
->number
- 1 ].dts
;
446 sample
->cts
= h264_imp
->ts_list
.timestamp
[ au
->number
- 1 ].cts
;
447 if( au
->number
< h264_imp
->num_undecodable
)
448 sample
->prop
.leading
= ISOM_SAMPLE_IS_UNDECODABLE_LEADING
;
450 sample
->prop
.leading
= picture
->independent
|| sample
->cts
>= h264_imp
->last_intra_cts
451 ? ISOM_SAMPLE_IS_NOT_LEADING
: ISOM_SAMPLE_IS_UNDECODABLE_LEADING
;
452 if( picture
->independent
)
453 h264_imp
->last_intra_cts
= sample
->cts
;
454 if( h264_imp
->composition_reordering_present
&& !picture
->disposable
&& !picture
->idr
)
455 sample
->prop
.allow_earlier
= QT_SAMPLE_EARLIER_PTS_ALLOWED
;
456 sample
->prop
.independent
= picture
->independent
? ISOM_SAMPLE_IS_INDEPENDENT
: ISOM_SAMPLE_IS_NOT_INDEPENDENT
;
457 sample
->prop
.disposable
= picture
->disposable
? ISOM_SAMPLE_IS_DISPOSABLE
: ISOM_SAMPLE_IS_NOT_DISPOSABLE
;
458 sample
->prop
.redundant
= picture
->has_redundancy
? ISOM_SAMPLE_HAS_REDUNDANCY
: ISOM_SAMPLE_HAS_NO_REDUNDANCY
;
459 sample
->prop
.post_roll
.identifier
= picture
->frame_num
;
460 if( picture
->random_accessible
)
463 sample
->prop
.ra_flags
= ISOM_SAMPLE_RANDOM_ACCESS_FLAG_SYNC
;
464 else if( picture
->recovery_frame_cnt
)
466 sample
->prop
.ra_flags
= ISOM_SAMPLE_RANDOM_ACCESS_FLAG_POST_ROLL_START
;
467 sample
->prop
.post_roll
.complete
= (picture
->frame_num
+ picture
->recovery_frame_cnt
) % info
->sps
.MaxFrameNum
;
471 sample
->prop
.ra_flags
= ISOM_SAMPLE_RANDOM_ACCESS_FLAG_RAP
;
472 if( !picture
->broken_link_flag
)
473 sample
->prop
.ra_flags
|= QT_SAMPLE_RANDOM_ACCESS_FLAG_PARTIAL_SYNC
;
476 sample
->length
= au
->length
;
477 memcpy( sample
->data
, au
->data
, au
->length
);
478 return current_status
;
481 static void nalu_deduplicate_poc
483 nal_pic_timing_t
*npt
,
484 uint32_t *max_composition_delay
,
485 uint32_t num_access_units
,
486 uint32_t max_num_reorder_pics
489 /* Deduplicate POCs. */
490 int64_t poc_offset
= 0;
492 int64_t invalid_poc_min
= 0;
493 uint32_t last_poc_reset
= UINT32_MAX
;
494 uint32_t invalid_poc_start
= 0;
495 int invalid_poc_present
= 0;
496 for( uint32_t i
= 0; ; i
++ )
498 if( i
< num_access_units
&& npt
[i
].poc
!= 0 && !npt
[i
].reset
)
500 /* poc_offset is not added to each POC here.
501 * It is done when we encounter the next coded video sequence. */
504 /* Pictures with negative POC shall precede IDR-picture in composition order.
505 * The minimum POC is added to poc_offset when we encounter the next coded video sequence. */
506 if( last_poc_reset
== UINT32_MAX
|| i
> last_poc_reset
+ max_num_reorder_pics
)
508 if( !invalid_poc_present
)
510 invalid_poc_present
= 1;
511 invalid_poc_start
= i
;
513 if( invalid_poc_min
> npt
[i
].poc
)
514 invalid_poc_min
= npt
[i
].poc
;
516 else if( poc_min
> npt
[i
].poc
)
518 poc_min
= npt
[i
].poc
;
519 *max_composition_delay
= LSMASH_MAX( *max_composition_delay
, i
- last_poc_reset
);
524 /* Encountered a new coded video sequence or no more POCs.
525 * Add poc_offset to each POC of the previous coded video sequence. */
526 poc_offset
-= poc_min
;
528 for( uint32_t j
= last_poc_reset
; j
< i
+ !!npt
[i
].reset
; j
++ )
529 if( npt
[j
].poc
>= 0 || (j
<= last_poc_reset
+ max_num_reorder_pics
) )
531 npt
[j
].poc
+= poc_offset
;
532 if( poc_max
< npt
[j
].poc
)
533 poc_max
= npt
[j
].poc
;
535 poc_offset
= poc_max
+ 1;
536 if( invalid_poc_present
)
538 /* Pictures with invalid negative POC is probably supposed to be composited
539 * both before the next coded video sequence and after the current one. */
540 poc_offset
-= invalid_poc_min
;
541 for( uint32_t j
= invalid_poc_start
; j
< i
+ !!npt
[i
].reset
; j
++ )
544 npt
[j
].poc
+= poc_offset
;
545 if( poc_max
< npt
[j
].poc
)
546 poc_max
= npt
[j
].poc
;
548 invalid_poc_present
= 0;
549 invalid_poc_start
= 0;
551 poc_offset
= poc_max
+ 1;
553 if( i
< num_access_units
)
561 break; /* no more POCs */
565 static void nalu_generate_timestamps_from_poc
567 importer_t
*importer
,
568 lsmash_media_ts_t
*timestamp
,
569 nal_pic_timing_t
*npt
,
570 uint8_t *composition_reordering_present
,
571 uint32_t *last_delta
,
572 uint32_t max_composition_delay
,
573 uint32_t num_access_units
576 /* Check if composition delay derived from reordering is present. */
577 if( max_composition_delay
== 0 )
579 *composition_reordering_present
= 0;
580 for( uint32_t i
= 1; i
< num_access_units
; i
++ )
581 if( npt
[i
].poc
< npt
[i
- 1].poc
)
583 *composition_reordering_present
= 1;
588 *composition_reordering_present
= 1;
589 if( *composition_reordering_present
)
591 /* Generate timestamps.
592 * Here, DTSs and CTSs are temporary values for sort. */
593 for( uint32_t i
= 0; i
< num_access_units
; i
++ )
595 timestamp
[i
].cts
= (uint64_t)npt
[i
].poc
;
596 timestamp
[i
].dts
= (uint64_t)i
;
598 qsort( timestamp
, num_access_units
, sizeof(lsmash_media_ts_t
), (int(*)( const void *, const void * ))lsmash_compare_cts
);
599 /* Check POC gap in output order. */
600 lsmash_class_t
*logger
= &(lsmash_class_t
){ .name
= importer
->class->name
};
601 for( uint32_t i
= 1; i
< num_access_units
; i
++ )
602 if( timestamp
[i
].cts
> timestamp
[i
- 1].cts
+ npt
[i
- 1].poc_delta
)
603 lsmash_log( &logger
, LSMASH_LOG_WARNING
,
604 "POC gap is detected at picture %"PRIu64
". Maybe some pictures are lost.\n", timestamp
[i
].dts
);
605 /* Get the maximum composition delay derived from reordering. */
606 for( uint32_t i
= 0; i
< num_access_units
; i
++ )
607 if( i
< timestamp
[i
].dts
)
609 uint32_t composition_delay
= timestamp
[i
].dts
- i
;
610 max_composition_delay
= LSMASH_MAX( max_composition_delay
, composition_delay
);
613 /* Generate timestamps. */
614 if( max_composition_delay
)
616 uint64_t *ts_buffer
= (uint64_t *)lsmash_malloc( (num_access_units
+ max_composition_delay
) * sizeof(uint64_t) );
619 /* It seems that there is no enough memory to generate more appropriate timestamps.
620 * Anyway, generate CTSs and DTSs. */
621 for( uint32_t i
= 0; i
< num_access_units
; i
++ )
622 timestamp
[i
].cts
= i
+ max_composition_delay
;
623 qsort( timestamp
, num_access_units
, sizeof(lsmash_media_ts_t
), (int(*)( const void *, const void * ))lsmash_compare_dts
);
627 uint64_t *reorder_cts
= ts_buffer
;
628 uint64_t *prev_reorder_cts
= ts_buffer
+ num_access_units
;
629 *last_delta
= npt
[num_access_units
- 1].delta
;
631 timestamp
[0].cts
= 0;
632 for( uint32_t i
= 1; i
< num_access_units
; i
++ )
633 timestamp
[i
].cts
= timestamp
[i
- 1].cts
+ npt
[i
- 1].delta
;
634 int64_t composition_delay_time
= timestamp
[max_composition_delay
].cts
;
635 for( uint32_t i
= 0; i
< num_access_units
; i
++ )
637 timestamp
[i
].cts
+= composition_delay_time
;
638 reorder_cts
[i
] = timestamp
[i
].cts
;
641 qsort( timestamp
, num_access_units
, sizeof(lsmash_media_ts_t
), (int(*)( const void *, const void * ))lsmash_compare_dts
);
642 for( uint32_t i
= 0; i
< num_access_units
; i
++ )
644 timestamp
[i
].dts
= i
<= max_composition_delay
645 ? reorder_cts
[i
] - composition_delay_time
646 : prev_reorder_cts
[(i
- max_composition_delay
) % max_composition_delay
];
647 prev_reorder_cts
[i
% max_composition_delay
] = reorder_cts
[i
];
649 lsmash_free( ts_buffer
);
651 fprintf( stderr
, "max_composition_delay=%"PRIu32
", composition_delay_time=%"PRIu64
"\n",
652 max_composition_delay
, composition_delay_time
);
657 timestamp
[0].dts
= 0;
658 timestamp
[0].cts
= 0;
659 for( uint32_t i
= 1; i
< num_access_units
; i
++ )
661 timestamp
[i
].dts
= timestamp
[i
- 1].dts
+ npt
[i
- 1].delta
;
662 timestamp
[i
].cts
= timestamp
[i
- 1].cts
+ npt
[i
- 1].delta
;
664 *last_delta
= npt
[num_access_units
- 1].delta
;
668 static void nalu_reduce_timescale
670 lsmash_media_ts_t
*timestamp
,
671 nal_pic_timing_t
*npt
,
672 uint32_t *last_delta
,
674 uint32_t num_access_units
677 uint64_t gcd_delta
= *timescale
;
678 for( uint32_t i
= 0; i
< num_access_units
&& gcd_delta
> 1; i
++ )
679 gcd_delta
= lsmash_get_gcd( gcd_delta
, npt
[i
].delta
);
682 for( uint32_t i
= 0; i
< num_access_units
; i
++ )
684 timestamp
[i
].dts
/= gcd_delta
;
685 timestamp
[i
].cts
/= gcd_delta
;
687 *last_delta
/= gcd_delta
;
688 *timescale
/= gcd_delta
;
691 for( uint32_t i
= 0; i
< num_access_units
; i
++ )
692 fprintf( stderr
, "Timestamp[%"PRIu32
"]: POC=%"PRId64
", DTS=%"PRIu64
", CTS=%"PRIu64
"\n",
693 i
, npt
[i
].poc
, timestamp
[i
].dts
, timestamp
[i
].cts
);
697 static lsmash_video_summary_t
*h264_setup_first_summary
702 h264_importer_t
*h264_imp
= (h264_importer_t
*)importer
->info
;
703 lsmash_codec_specific_t
*cs
= (lsmash_codec_specific_t
*)lsmash_get_entry_data( h264_imp
->avcC_list
, ++ h264_imp
->avcC_number
);
704 if( !cs
|| !cs
->data
.structured
)
706 lsmash_destroy_codec_specific_data( cs
);
709 lsmash_video_summary_t
*summary
= h264_create_summary( (lsmash_h264_specific_parameters_t
*)cs
->data
.structured
,
710 &h264_imp
->info
.sps
, h264_imp
->max_au_length
);
713 lsmash_destroy_codec_specific_data( cs
);
716 if( lsmash_add_entry( importer
->summaries
, summary
) < 0 )
718 lsmash_cleanup_summary( (lsmash_summary_t
*)summary
);
721 summary
->sample_per_field
= h264_imp
->field_pic_present
;
725 static int h264_analyze_whole_stream
730 /* Parse all NALU in the stream for preparation of calculating timestamps. */
731 uint32_t npt_alloc
= (1 << 12) * sizeof(nal_pic_timing_t
);
732 nal_pic_timing_t
*npt
= lsmash_malloc( npt_alloc
);
734 return LSMASH_ERR_MEMORY_ALLOC
;
735 uint32_t picture_stats
[H264_PICTURE_TYPE_NONE
+ 1] = { 0 };
736 uint32_t num_access_units
= 0;
737 lsmash_class_t
*logger
= &(lsmash_class_t
){ "H.264" };
738 lsmash_log( &logger
, LSMASH_LOG_INFO
, "Analyzing stream as H.264\r" );
739 h264_importer_t
*h264_imp
= (h264_importer_t
*)importer
->info
;
740 h264_info_t
*info
= &h264_imp
->info
;
741 importer
->status
= IMPORTER_OK
;
742 int err
= LSMASH_ERR_MEMORY_ALLOC
;
743 while( importer
->status
!= IMPORTER_EOF
)
746 lsmash_log( &logger
, LSMASH_LOG_INFO
, "Analyzing stream as H.264: %"PRIu32
"\n", num_access_units
+ 1 );
748 h264_picture_info_t
*picture
= &info
->au
.picture
;
749 h264_picture_info_t prev_picture
= *picture
;
750 if( (err
= h264_get_access_unit_internal( importer
, 1 )) < 0
751 || (err
= h264_calculate_poc( info
, picture
, &prev_picture
)) < 0 )
753 h264_importer_check_eof( importer
, &info
->au
);
754 if( npt_alloc
<= num_access_units
* sizeof(nal_pic_timing_t
) )
756 uint32_t alloc
= 2 * num_access_units
* sizeof(nal_pic_timing_t
);
757 nal_pic_timing_t
*temp
= (nal_pic_timing_t
*)lsmash_realloc( npt
, alloc
);
763 h264_imp
->field_pic_present
|= picture
->field_pic_flag
;
764 npt
[num_access_units
].poc
= picture
->PicOrderCnt
;
765 npt
[num_access_units
].delta
= picture
->delta
;
766 npt
[num_access_units
].poc_delta
= picture
->field_pic_flag
? 1 : 2;
767 npt
[num_access_units
].reset
= picture
->has_mmco5
;
769 h264_imp
->max_au_length
= LSMASH_MAX( info
->au
.length
, h264_imp
->max_au_length
);
771 ++picture_stats
[H264_PICTURE_TYPE_IDR
];
772 else if( picture
->type
>= H264_PICTURE_TYPE_NONE
)
773 ++picture_stats
[H264_PICTURE_TYPE_NONE
];
775 ++picture_stats
[ picture
->type
];
777 lsmash_log_refresh_line( &logger
);
778 lsmash_log( &logger
, LSMASH_LOG_INFO
,
779 "IDR: %"PRIu32
", I: %"PRIu32
", P: %"PRIu32
", B: %"PRIu32
", "
780 "SI: %"PRIu32
", SP: %"PRIu32
", Unknown: %"PRIu32
"\n",
781 picture_stats
[H264_PICTURE_TYPE_IDR
],
782 picture_stats
[H264_PICTURE_TYPE_I
],
783 picture_stats
[H264_PICTURE_TYPE_I_P
],
784 picture_stats
[H264_PICTURE_TYPE_I_P_B
],
785 picture_stats
[H264_PICTURE_TYPE_SI
]
786 + picture_stats
[H264_PICTURE_TYPE_I_SI
],
787 picture_stats
[H264_PICTURE_TYPE_SI_SP
]
788 + picture_stats
[H264_PICTURE_TYPE_I_SI_P_SP
]
789 + picture_stats
[H264_PICTURE_TYPE_I_SI_P_SP_B
],
790 picture_stats
[H264_PICTURE_TYPE_NONE
] );
791 /* Copy and append the last Codec Specific info. */
792 if( (err
= h264_store_codec_specific( h264_imp
, &info
->avcC_param
)) < 0 )
794 /* Set up the first summary. */
795 lsmash_video_summary_t
*summary
= h264_setup_first_summary( importer
);
798 err
= LSMASH_ERR_NAMELESS
;
801 /* Allocate timestamps. */
802 lsmash_media_ts_t
*timestamp
= lsmash_malloc( num_access_units
* sizeof(lsmash_media_ts_t
) );
805 /* Count leading samples that are undecodable. */
806 for( uint32_t i
= 0; i
< num_access_units
; i
++ )
808 if( npt
[i
].poc
== 0 )
810 ++ h264_imp
->num_undecodable
;
812 /* Deduplicate POCs. */
813 uint32_t max_composition_delay
= 0;
814 nalu_deduplicate_poc( npt
, &max_composition_delay
, num_access_units
, 32 );
815 /* Generate timestamps. */
816 nalu_generate_timestamps_from_poc( importer
, timestamp
, npt
,
817 &h264_imp
->composition_reordering_present
,
818 &h264_imp
->last_delta
,
819 max_composition_delay
, num_access_units
);
820 nalu_reduce_timescale( timestamp
, npt
, &h264_imp
->last_delta
, &summary
->timescale
, num_access_units
);
822 h264_imp
->ts_list
.sample_count
= num_access_units
;
823 h264_imp
->ts_list
.timestamp
= timestamp
;
826 lsmash_log_refresh_line( &logger
);
831 static int h264_importer_probe( importer_t
*importer
)
833 /* Find the first start code. */
834 h264_importer_t
*h264_imp
= create_h264_importer( importer
);
836 return LSMASH_ERR_MEMORY_ALLOC
;
837 lsmash_bs_t
*bs
= importer
->bs
;
838 uint64_t first_sc_head_pos
= nalu_find_first_start_code( bs
);
840 if( first_sc_head_pos
== NALU_NO_START_CODE_FOUND
)
842 err
= LSMASH_ERR_INVALID_DATA
;
845 else if( first_sc_head_pos
== NALU_IO_ERROR
)
850 /* OK. It seems the stream has a long start code of H.264. */
851 importer
->info
= h264_imp
;
852 h264_info_t
*info
= &h264_imp
->info
;
853 lsmash_bs_read_seek( bs
, first_sc_head_pos
, SEEK_SET
);
854 h264_imp
->sc_head_pos
= first_sc_head_pos
;
855 if( (err
= h264_analyze_whole_stream( importer
)) < 0 )
857 /* Go back to the start code of the first NALU. */
858 importer
->status
= IMPORTER_OK
;
859 lsmash_bs_read_seek( bs
, first_sc_head_pos
, SEEK_SET
);
860 h264_imp
->sc_head_pos
= first_sc_head_pos
;
861 info
->prev_nalu_type
= H264_NALU_TYPE_UNSPECIFIED0
;
862 uint8_t *temp_au
= info
->au
.data
;
863 uint8_t *temp_incomplete_au
= info
->au
.incomplete_data
;
864 memset( &info
->au
, 0, sizeof(h264_access_unit_t
) );
865 info
->au
.data
= temp_au
;
866 info
->au
.incomplete_data
= temp_incomplete_au
;
867 memset( &info
->slice
, 0, sizeof(h264_slice_info_t
) );
868 memset( &info
->sps
, 0, sizeof(h264_sps_t
) );
869 memset( &info
->pps
, 0, sizeof(h264_pps_t
) );
870 lsmash_remove_entries( info
->avcC_param
.parameter_sets
->sps_list
, isom_remove_dcr_ps
);
871 lsmash_remove_entries( info
->avcC_param
.parameter_sets
->pps_list
, isom_remove_dcr_ps
);
872 lsmash_remove_entries( info
->avcC_param
.parameter_sets
->spsext_list
, isom_remove_dcr_ps
);
873 lsmash_destroy_h264_parameter_sets( &info
->avcC_param_next
);
876 remove_h264_importer( h264_imp
);
877 importer
->info
= NULL
;
878 lsmash_remove_entries( importer
->summaries
, lsmash_cleanup_summary
);
882 static uint32_t h264_importer_get_last_delta( importer_t
*importer
, uint32_t track_number
)
884 debug_if( !importer
|| !importer
->info
)
886 h264_importer_t
*h264_imp
= (h264_importer_t
*)importer
->info
;
887 if( !h264_imp
|| track_number
!= 1 || importer
->status
!= IMPORTER_EOF
)
889 return h264_imp
->ts_list
.sample_count
890 ? h264_imp
->last_delta
891 : UINT32_MAX
; /* arbitrary */
894 const importer_functions h264_importer
=
896 { "H.264", offsetof( importer_t
, log_level
) },
899 h264_importer_get_accessunit
,
900 h264_importer_get_last_delta
,
901 h264_importer_cleanup
904 /***************************************************************************
906 ITU-T Recommendation H.265 (04/13)
907 ISO/IEC 14496-15:2014
908 ***************************************************************************/
909 #include "codecs/hevc.h"
914 lsmash_entry_list_t hvcC_list
[1]; /* stored as lsmash_codec_specific_t */
915 lsmash_media_ts_list_t ts_list
;
916 uint32_t max_au_length
;
917 uint32_t num_undecodable
;
918 uint32_t hvcC_number
;
920 uint64_t last_intra_cts
;
921 uint64_t sc_head_pos
;
922 uint8_t composition_reordering_present
;
923 uint8_t field_pic_present
;
924 uint8_t max_TemporalId
;
927 static void remove_hevc_importer( hevc_importer_t
*hevc_imp
)
931 lsmash_remove_entries( hevc_imp
->hvcC_list
, lsmash_destroy_codec_specific_data
);
932 hevc_cleanup_parser( &hevc_imp
->info
);
933 lsmash_free( hevc_imp
->ts_list
.timestamp
);
934 lsmash_free( hevc_imp
);
937 static void hevc_importer_cleanup( importer_t
*importer
)
939 debug_if( importer
&& importer
->info
)
940 remove_hevc_importer( importer
->info
);
943 static hevc_importer_t
*create_hevc_importer( importer_t
*importer
)
945 hevc_importer_t
*hevc_imp
= lsmash_malloc_zero( sizeof(hevc_importer_t
) );
948 if( hevc_setup_parser( &hevc_imp
->info
, 0 ) < 0 )
950 remove_hevc_importer( hevc_imp
);
953 lsmash_init_entry_list( hevc_imp
->hvcC_list
);
954 hevc_imp
->info
.eos
= 1;
958 static inline int hevc_complete_au( hevc_access_unit_t
*au
, int probe
)
960 if( !au
->picture
.has_primary
|| au
->incomplete_length
== 0 )
963 memcpy( au
->data
, au
->incomplete_data
, au
->incomplete_length
);
964 au
->TemporalId
= au
->picture
.TemporalId
;
965 au
->length
= au
->incomplete_length
;
966 au
->incomplete_length
= 0;
967 au
->picture
.has_primary
= 0;
971 static void hevc_append_nalu_to_au( hevc_access_unit_t
*au
, uint8_t *src_nalu
, uint32_t nalu_length
, int probe
)
975 uint8_t *dst_nalu
= au
->incomplete_data
+ au
->incomplete_length
+ NALU_DEFAULT_NALU_LENGTH_SIZE
;
976 for( int i
= NALU_DEFAULT_NALU_LENGTH_SIZE
; i
; i
-- )
977 *(dst_nalu
- i
) = (nalu_length
>> ((i
- 1) * 8)) & 0xff;
978 memcpy( dst_nalu
, src_nalu
, nalu_length
);
980 /* Note: picture->incomplete_au_length shall be 0 immediately after AU has completed.
981 * Therefore, possible_au_length in hevc_get_access_unit_internal() can't be used here
982 * to avoid increasing AU length monotonously through the entire stream. */
983 au
->incomplete_length
+= NALU_DEFAULT_NALU_LENGTH_SIZE
+ nalu_length
;
986 static int hevc_get_au_internal_succeeded( hevc_importer_t
*hevc_imp
, hevc_access_unit_t
*au
)
992 static int hevc_get_au_internal_failed( hevc_importer_t
*hevc_imp
, hevc_access_unit_t
*au
, int complete_au
, int ret
)
999 static lsmash_video_summary_t
*hevc_create_summary
1001 lsmash_hevc_specific_parameters_t
*param
,
1003 uint32_t max_au_length
1006 lsmash_video_summary_t
*summary
= (lsmash_video_summary_t
*)lsmash_create_summary( LSMASH_SUMMARY_TYPE_VIDEO
);
1009 /* Update summary here.
1010 * max_au_length is set at the last of hevc_importer_probe function. */
1011 lsmash_codec_specific_t
*specific
= lsmash_create_codec_specific_data( LSMASH_CODEC_SPECIFIC_DATA_TYPE_ISOM_VIDEO_HEVC
,
1012 LSMASH_CODEC_SPECIFIC_FORMAT_UNSTRUCTURED
);
1015 lsmash_cleanup_summary( (lsmash_summary_t
*)summary
);
1018 specific
->data
.unstructured
= lsmash_create_hevc_specific_info( param
, &specific
->size
);
1019 if( !specific
->data
.unstructured
1020 || lsmash_add_entry( &summary
->opaque
->list
, specific
) < 0 )
1022 lsmash_cleanup_summary( (lsmash_summary_t
*)summary
);
1023 lsmash_destroy_codec_specific_data( specific
);
1026 summary
->sample_type
= ISOM_CODEC_TYPE_HVC1_VIDEO
;
1027 summary
->max_au_length
= max_au_length
;
1028 summary
->timescale
= sps
->vui
.time_scale
;
1029 summary
->timebase
= sps
->vui
.num_units_in_tick
;
1030 summary
->vfr
= (param
->constantFrameRate
== 0);
1031 summary
->sample_per_field
= 0;
1032 summary
->width
= sps
->cropped_width
;
1033 summary
->height
= sps
->cropped_height
;
1034 summary
->par_h
= sps
->vui
.sar_width
;
1035 summary
->par_v
= sps
->vui
.sar_height
;
1036 summary
->color
.primaries_index
= sps
->vui
.colour_primaries
!= 2 ? sps
->vui
.colour_primaries
: 0;
1037 summary
->color
.transfer_index
= sps
->vui
.transfer_characteristics
!= 2 ? sps
->vui
.transfer_characteristics
: 0;
1038 summary
->color
.matrix_index
= sps
->vui
.matrix_coeffs
!= 2 ? sps
->vui
.matrix_coeffs
: 0;
1039 summary
->color
.full_range
= sps
->vui
.video_full_range_flag
;
1040 lsmash_convert_crop_into_clap( sps
->vui
.def_disp_win_offset
, summary
->width
, summary
->height
, &summary
->clap
);
1044 static int hevc_store_codec_specific
1046 hevc_importer_t
*hevc_imp
,
1047 lsmash_hevc_specific_parameters_t
*hvcC_param
1050 lsmash_codec_specific_t
*src_cs
= lsmash_create_codec_specific_data( LSMASH_CODEC_SPECIFIC_DATA_TYPE_ISOM_VIDEO_HEVC
,
1051 LSMASH_CODEC_SPECIFIC_FORMAT_STRUCTURED
);
1053 return LSMASH_ERR_NAMELESS
;
1054 lsmash_hevc_specific_parameters_t
*src_param
= (lsmash_hevc_specific_parameters_t
*)src_cs
->data
.structured
;
1055 *src_param
= *hvcC_param
;
1056 lsmash_codec_specific_t
*dst_cs
= lsmash_convert_codec_specific_format( src_cs
, LSMASH_CODEC_SPECIFIC_FORMAT_STRUCTURED
);
1057 src_param
->parameter_arrays
= NULL
; /* Avoid freeing parameter arrays within hvcC_param. */
1058 lsmash_destroy_codec_specific_data( src_cs
);
1061 lsmash_destroy_codec_specific_data( dst_cs
);
1062 return LSMASH_ERR_NAMELESS
;
1064 if( lsmash_add_entry( hevc_imp
->hvcC_list
, dst_cs
) < 0 )
1066 lsmash_destroy_codec_specific_data( dst_cs
);
1067 return LSMASH_ERR_MEMORY_ALLOC
;
1072 static inline void hevc_new_access_unit( hevc_access_unit_t
*au
)
1075 au
->picture
.type
= HEVC_PICTURE_TYPE_NONE
;
1076 au
->picture
.random_accessible
= 0;
1077 au
->picture
.recovery_poc_cnt
= 0;
1080 /* If probe equals 0, don't get the actual data (EBPS) of an access unit and only parse NALU. */
1081 static int hevc_get_access_unit_internal
1083 importer_t
*importer
,
1087 hevc_importer_t
*hevc_imp
= (hevc_importer_t
*)importer
->info
;
1088 hevc_info_t
*info
= &hevc_imp
->info
;
1089 hevc_slice_info_t
*slice
= &info
->slice
;
1090 hevc_access_unit_t
*au
= &info
->au
;
1091 hevc_picture_info_t
*picture
= &au
->picture
;
1092 hevc_stream_buffer_t
*sb
= &info
->buffer
;
1093 lsmash_bs_t
*bs
= importer
->bs
;
1094 int complete_au
= 0;
1095 hevc_new_access_unit( au
);
1098 hevc_nalu_header_t nuh
;
1099 uint64_t start_code_length
;
1100 uint64_t trailing_zero_bytes
;
1101 uint64_t nalu_length
= hevc_find_next_start_code( bs
, &nuh
, &start_code_length
, &trailing_zero_bytes
);
1102 if( nalu_length
== NALU_NO_START_CODE_FOUND
)
1104 /* For the last NALU.
1105 * This NALU already has been appended into the latest access unit and parsed. */
1106 hevc_update_picture_info( info
, picture
, slice
, &info
->sps
, &info
->sei
);
1107 complete_au
= hevc_complete_au( au
, probe
);
1109 return hevc_get_au_internal_succeeded( hevc_imp
, au
);
1111 return hevc_get_au_internal_failed( hevc_imp
, au
, complete_au
, LSMASH_ERR_INVALID_DATA
);
1113 uint8_t nalu_type
= nuh
.nal_unit_type
;
1114 uint64_t next_sc_head_pos
= hevc_imp
->sc_head_pos
1117 + trailing_zero_bytes
;
1121 fprintf( stderr
, "NALU type: %"PRIu8
" \n", nalu_type
);
1122 fprintf( stderr
, " NALU header position: %"PRIx64
" \n", hevc_imp
->sc_head_pos
+ start_code_length
);
1123 fprintf( stderr
, " EBSP position: %"PRIx64
" \n", hevc_imp
->sc_head_pos
+ start_code_length
+ nuh
.length
);
1124 fprintf( stderr
, " EBSP length: %"PRIx64
" (%"PRIu64
") \n", nalu_length
- nuh
.length
, nalu_length
- nuh
.length
);
1125 fprintf( stderr
, " trailing_zero_bytes: %"PRIx64
" \n", trailing_zero_bytes
);
1126 fprintf( stderr
, " Next start code position: %"PRIx64
"\n", next_sc_head_pos
);
1129 /* Check if the end of sequence. Used for POC calculation. */
1130 info
->eos
|= info
->prev_nalu_type
== HEVC_NALU_TYPE_EOS
1131 || info
->prev_nalu_type
== HEVC_NALU_TYPE_EOB
;
1132 /* Process the current NALU by its type. */
1133 if( nalu_type
== HEVC_NALU_TYPE_FD
)
1135 /* We don't support streams with both filler and HRD yet. Otherwise, just skip filler. */
1136 if( info
->sps
.vui
.hrd
.present
)
1137 return hevc_get_au_internal_failed( hevc_imp
, au
, complete_au
, LSMASH_ERR_PATCH_WELCOME
);
1139 else if( nalu_type
<= HEVC_NALU_TYPE_RASL_R
1140 || (nalu_type
>= HEVC_NALU_TYPE_BLA_W_LP
&& nalu_type
<= HEVC_NALU_TYPE_CRA
)
1141 || (nalu_type
>= HEVC_NALU_TYPE_VPS
&& nalu_type
<= HEVC_NALU_TYPE_SUFFIX_SEI
) )
1144 /* Increase the buffer if needed. */
1145 uint64_t possible_au_length
= au
->incomplete_length
+ NALU_DEFAULT_NALU_LENGTH_SIZE
+ nalu_length
;
1146 if( sb
->bank
->buffer_size
< possible_au_length
1147 && (err
= hevc_supplement_buffer( sb
, au
, 2 * possible_au_length
)) < 0 )
1149 lsmash_log( importer
, LSMASH_LOG_ERROR
, "failed to increase the buffer size.\n" );
1150 return hevc_get_au_internal_failed( hevc_imp
, au
, complete_au
, err
);
1152 /* Get the EBSP of the current NALU here. */
1153 uint8_t *nalu
= lsmash_bs_get_buffer_data( bs
) + start_code_length
;
1154 if( nalu_type
<= HEVC_NALU_TYPE_RSV_VCL31
)
1156 /* VCL NALU (slice) */
1157 hevc_slice_info_t prev_slice
= *slice
;
1158 if( (err
= hevc_parse_slice_segment_header( info
, &nuh
, sb
->rbsp
,
1160 nalu_length
- nuh
.length
)) < 0 )
1161 return hevc_get_au_internal_failed( hevc_imp
, au
, complete_au
, err
);
1162 if( probe
&& info
->hvcC_pending
)
1164 /* Copy and append a Codec Specific info. */
1165 if( (err
= hevc_store_codec_specific( hevc_imp
, &info
->hvcC_param
)) < 0 )
1168 if( (err
= hevc_move_pending_hvcC_param( info
)) < 0 )
1170 if( prev_slice
.present
)
1172 /* Check whether the AU that contains the previous VCL NALU completed or not. */
1173 if( hevc_find_au_delimit_by_slice_info( info
, slice
, &prev_slice
) )
1175 /* The current NALU is the first VCL NALU of the primary coded picture of a new AU.
1176 * Therefore, the previous slice belongs to the AU you want at this time. */
1177 hevc_update_picture_info( info
, picture
, &prev_slice
, &info
->sps
, &info
->sei
);
1178 complete_au
= hevc_complete_au( au
, probe
);
1181 hevc_update_picture_info_for_slice( info
, picture
, &prev_slice
);
1183 hevc_append_nalu_to_au( au
, nalu
, nalu_length
, probe
);
1188 if( hevc_find_au_delimit_by_nalu_type( nalu_type
, info
->prev_nalu_type
) )
1190 /* The last slice belongs to the AU you want at this time. */
1191 hevc_update_picture_info( info
, picture
, slice
, &info
->sps
, &info
->sei
);
1192 complete_au
= hevc_complete_au( au
, probe
);
1196 case HEVC_NALU_TYPE_PREFIX_SEI
:
1197 case HEVC_NALU_TYPE_SUFFIX_SEI
:
1199 if( (err
= hevc_parse_sei( info
->bits
, &info
->vps
, &info
->sps
, &info
->sei
, &nuh
,
1200 sb
->rbsp
, nalu
+ nuh
.length
, nalu_length
- nuh
.length
)) < 0 )
1201 return hevc_get_au_internal_failed( hevc_imp
, au
, complete_au
, err
);
1202 hevc_append_nalu_to_au( au
, nalu
, nalu_length
, probe
);
1205 case HEVC_NALU_TYPE_VPS
:
1206 if( (err
= hevc_try_to_append_dcr_nalu( info
, HEVC_DCR_NALU_TYPE_VPS
, nalu
, nalu_length
)) < 0 )
1207 return hevc_get_au_internal_failed( hevc_imp
, au
, complete_au
, err
);
1209 case HEVC_NALU_TYPE_SPS
:
1210 if( (err
= hevc_try_to_append_dcr_nalu( info
, HEVC_DCR_NALU_TYPE_SPS
, nalu
, nalu_length
)) < 0 )
1211 return hevc_get_au_internal_failed( hevc_imp
, au
, complete_au
, err
);
1213 case HEVC_NALU_TYPE_PPS
:
1214 if( (err
= hevc_try_to_append_dcr_nalu( info
, HEVC_DCR_NALU_TYPE_PPS
, nalu
, nalu_length
)) < 0 )
1215 return hevc_get_au_internal_failed( hevc_imp
, au
, complete_au
, err
);
1217 case HEVC_NALU_TYPE_AUD
: /* We drop access unit delimiters. */
1220 hevc_append_nalu_to_au( au
, nalu
, nalu_length
, probe
);
1223 if( info
->hvcC_pending
)
1224 importer
->status
= IMPORTER_CHANGE
;
1227 /* Move to the first byte of the next start code. */
1228 info
->prev_nalu_type
= nalu_type
;
1229 if( lsmash_bs_read_seek( bs
, next_sc_head_pos
, SEEK_SET
) != next_sc_head_pos
)
1231 lsmash_log( importer
, LSMASH_LOG_ERROR
, "failed to seek the next start code.\n" );
1232 return hevc_get_au_internal_failed( hevc_imp
, au
, complete_au
, LSMASH_ERR_NAMELESS
);
1234 if( !lsmash_bs_is_end( bs
, NALU_SHORT_START_CODE_LENGTH
) )
1235 hevc_imp
->sc_head_pos
= next_sc_head_pos
;
1236 /* If there is no more data in the stream, and flushed chunk of NALUs, flush it as complete AU here. */
1237 else if( au
->incomplete_length
&& au
->length
== 0 )
1239 hevc_update_picture_info( info
, picture
, slice
, &info
->sps
, &info
->sei
);
1240 hevc_complete_au( au
, probe
);
1241 return hevc_get_au_internal_succeeded( hevc_imp
, au
);
1244 return hevc_get_au_internal_succeeded( hevc_imp
, au
);
1248 static inline void hevc_importer_check_eof( importer_t
*importer
, hevc_access_unit_t
*au
)
1250 /* HEVC byte stream NALU consists of at least 5 bytes (start-code + NALU-header). */
1251 if( lsmash_bs_is_end( importer
->bs
, NALU_SHORT_START_CODE_LENGTH
+ 1 ) && au
->incomplete_length
== 0 )
1252 importer
->status
= IMPORTER_EOF
;
1253 else if( importer
->status
!= IMPORTER_CHANGE
)
1254 importer
->status
= IMPORTER_OK
;
1257 static int hevc_importer_get_accessunit( importer_t
*importer
, uint32_t track_number
, lsmash_sample_t
**p_sample
)
1259 if( !importer
->info
)
1260 return LSMASH_ERR_NAMELESS
;
1261 if( track_number
!= 1 )
1262 return LSMASH_ERR_FUNCTION_PARAM
;
1263 hevc_importer_t
*hevc_imp
= (hevc_importer_t
*)importer
->info
;
1264 hevc_info_t
*info
= &hevc_imp
->info
;
1265 importer_status current_status
= importer
->status
;
1266 if( current_status
== IMPORTER_ERROR
)
1267 return LSMASH_ERR_NAMELESS
;
1268 if( current_status
== IMPORTER_EOF
)
1269 return IMPORTER_EOF
;
1270 int err
= hevc_get_access_unit_internal( importer
, 0 );
1273 importer
->status
= IMPORTER_ERROR
;
1276 hevc_importer_check_eof( importer
, &info
->au
);
1277 if( importer
->status
== IMPORTER_CHANGE
&& !info
->hvcC_pending
)
1278 current_status
= IMPORTER_CHANGE
;
1279 if( current_status
== IMPORTER_CHANGE
)
1281 /* Update the active summary. */
1282 lsmash_codec_specific_t
*cs
= (lsmash_codec_specific_t
*)lsmash_get_entry_data( hevc_imp
->hvcC_list
, ++ hevc_imp
->hvcC_number
);
1284 return LSMASH_ERR_NAMELESS
;
1285 lsmash_hevc_specific_parameters_t
*hvcC_param
= (lsmash_hevc_specific_parameters_t
*)cs
->data
.structured
;
1286 lsmash_video_summary_t
*summary
= hevc_create_summary( hvcC_param
, &info
->sps
, hevc_imp
->max_au_length
);
1288 return LSMASH_ERR_NAMELESS
;
1289 lsmash_remove_entry( importer
->summaries
, track_number
, lsmash_cleanup_summary
);
1290 if( lsmash_add_entry( importer
->summaries
, summary
) < 0 )
1292 lsmash_cleanup_summary( (lsmash_summary_t
*)summary
);
1293 return LSMASH_ERR_MEMORY_ALLOC
;
1295 importer
->status
= IMPORTER_OK
;
1297 lsmash_sample_t
*sample
= lsmash_create_sample( hevc_imp
->max_au_length
);
1299 return LSMASH_ERR_MEMORY_ALLOC
;
1301 hevc_access_unit_t
*au
= &info
->au
;
1302 hevc_picture_info_t
*picture
= &au
->picture
;
1303 sample
->dts
= hevc_imp
->ts_list
.timestamp
[ au
->number
- 1 ].dts
;
1304 sample
->cts
= hevc_imp
->ts_list
.timestamp
[ au
->number
- 1 ].cts
;
1305 /* Set property of disposability. */
1306 if( picture
->sublayer_nonref
&& au
->TemporalId
== hevc_imp
->max_TemporalId
)
1307 /* Sub-layer non-reference pictures are not referenced by subsequent pictures of
1308 * the same sub-layer in decoding order. */
1309 sample
->prop
.disposable
= ISOM_SAMPLE_IS_DISPOSABLE
;
1311 sample
->prop
.disposable
= ISOM_SAMPLE_IS_NOT_DISPOSABLE
;
1312 /* Set property of leading. */
1313 if( picture
->radl
|| picture
->rasl
)
1314 sample
->prop
.leading
= picture
->radl
? ISOM_SAMPLE_IS_DECODABLE_LEADING
: ISOM_SAMPLE_IS_UNDECODABLE_LEADING
;
1317 if( au
->number
< hevc_imp
->num_undecodable
)
1318 sample
->prop
.leading
= ISOM_SAMPLE_IS_UNDECODABLE_LEADING
;
1321 if( picture
->independent
|| sample
->cts
>= hevc_imp
->last_intra_cts
)
1322 sample
->prop
.leading
= ISOM_SAMPLE_IS_NOT_LEADING
;
1324 sample
->prop
.leading
= ISOM_SAMPLE_IS_UNDECODABLE_LEADING
;
1327 if( picture
->independent
)
1328 hevc_imp
->last_intra_cts
= sample
->cts
;
1329 /* Set property of independence. */
1330 sample
->prop
.independent
= picture
->independent
? ISOM_SAMPLE_IS_INDEPENDENT
: ISOM_SAMPLE_IS_NOT_INDEPENDENT
;
1331 sample
->prop
.redundant
= ISOM_SAMPLE_HAS_NO_REDUNDANCY
;
1332 sample
->prop
.post_roll
.identifier
= picture
->poc
;
1333 if( picture
->random_accessible
)
1337 sample
->prop
.ra_flags
= ISOM_SAMPLE_RANDOM_ACCESS_FLAG_SYNC
;
1338 if( picture
->closed_rap
)
1339 sample
->prop
.ra_flags
|= ISOM_SAMPLE_RANDOM_ACCESS_FLAG_CLOSED_RAP
;
1341 sample
->prop
.ra_flags
|= ISOM_SAMPLE_RANDOM_ACCESS_FLAG_RAP
;
1343 else if( picture
->recovery_poc_cnt
)
1345 sample
->prop
.ra_flags
= ISOM_SAMPLE_RANDOM_ACCESS_FLAG_POST_ROLL_START
;
1346 sample
->prop
.post_roll
.complete
= picture
->poc
+ picture
->recovery_poc_cnt
;
1349 sample
->prop
.ra_flags
= ISOM_SAMPLE_RANDOM_ACCESS_FLAG_RAP
;
1351 sample
->length
= au
->length
;
1352 memcpy( sample
->data
, au
->data
, au
->length
);
1353 return current_status
;
1356 static lsmash_video_summary_t
*hevc_setup_first_summary
1358 importer_t
*importer
1361 hevc_importer_t
*hevc_imp
= (hevc_importer_t
*)importer
->info
;
1362 lsmash_codec_specific_t
*cs
= (lsmash_codec_specific_t
*)lsmash_get_entry_data( hevc_imp
->hvcC_list
, ++ hevc_imp
->hvcC_number
);
1363 if( !cs
|| !cs
->data
.structured
)
1365 lsmash_destroy_codec_specific_data( cs
);
1368 lsmash_video_summary_t
*summary
= hevc_create_summary( (lsmash_hevc_specific_parameters_t
*)cs
->data
.structured
,
1369 &hevc_imp
->info
.sps
, hevc_imp
->max_au_length
);
1372 lsmash_destroy_codec_specific_data( cs
);
1375 if( lsmash_add_entry( importer
->summaries
, summary
) < 0 )
1377 lsmash_cleanup_summary( (lsmash_summary_t
*)summary
);
1380 summary
->sample_per_field
= hevc_imp
->field_pic_present
;
1384 static int hevc_analyze_whole_stream
1386 importer_t
*importer
1389 /* Parse all NALU in the stream for preparation of calculating timestamps. */
1390 uint32_t npt_alloc
= (1 << 12) * sizeof(nal_pic_timing_t
);
1391 nal_pic_timing_t
*npt
= (nal_pic_timing_t
*)lsmash_malloc( npt_alloc
);
1393 return LSMASH_ERR_MEMORY_ALLOC
;
1394 uint32_t picture_stats
[HEVC_PICTURE_TYPE_NONE
+ 1] = { 0 };
1395 uint32_t num_access_units
= 0;
1396 lsmash_class_t
*logger
= &(lsmash_class_t
){ "HEVC" };
1397 lsmash_log( &logger
, LSMASH_LOG_INFO
, "Analyzing stream as HEVC\r" );
1398 hevc_importer_t
*hevc_imp
= (hevc_importer_t
*)importer
->info
;
1399 hevc_info_t
*info
= &hevc_imp
->info
;
1400 importer
->status
= IMPORTER_OK
;
1401 int err
= LSMASH_ERR_MEMORY_ALLOC
;
1402 while( importer
->status
!= IMPORTER_EOF
)
1405 lsmash_log( &logger
, LSMASH_LOG_INFO
, "Analyzing stream as HEVC: %"PRIu32
"\n", num_access_units
+ 1 );
1407 hevc_picture_info_t
*picture
= &info
->au
.picture
;
1408 hevc_picture_info_t prev_picture
= *picture
;
1409 if( (err
= hevc_get_access_unit_internal( importer
, 1 )) < 0
1410 || (err
= hevc_calculate_poc( info
, &info
->au
.picture
, &prev_picture
)) < 0 )
1412 hevc_importer_check_eof( importer
, &info
->au
);
1413 if( npt_alloc
<= num_access_units
* sizeof(nal_pic_timing_t
) )
1415 uint32_t alloc
= 2 * num_access_units
* sizeof(nal_pic_timing_t
);
1416 nal_pic_timing_t
*temp
= (nal_pic_timing_t
*)lsmash_realloc( npt
, alloc
);
1422 hevc_imp
->field_pic_present
|= picture
->field_coded
;
1423 npt
[num_access_units
].poc
= picture
->poc
;
1424 npt
[num_access_units
].delta
= picture
->delta
;
1425 npt
[num_access_units
].poc_delta
= 1;
1426 npt
[num_access_units
].reset
= 0;
1428 hevc_imp
->max_au_length
= LSMASH_MAX( hevc_imp
->max_au_length
, info
->au
.length
);
1429 hevc_imp
->max_TemporalId
= LSMASH_MAX( hevc_imp
->max_TemporalId
, info
->au
.TemporalId
);
1431 ++picture_stats
[HEVC_PICTURE_TYPE_IDR
];
1432 else if( picture
->irap
)
1433 ++picture_stats
[ picture
->broken_link
? HEVC_PICTURE_TYPE_BLA
: HEVC_PICTURE_TYPE_CRA
];
1434 else if( picture
->type
>= HEVC_PICTURE_TYPE_NONE
)
1435 ++picture_stats
[HEVC_PICTURE_TYPE_NONE
];
1437 ++picture_stats
[ picture
->type
];
1439 lsmash_log_refresh_line( &logger
);
1440 lsmash_log( &logger
, LSMASH_LOG_INFO
,
1441 "IDR: %"PRIu32
", CRA: %"PRIu32
", BLA: %"PRIu32
", I: %"PRIu32
", P: %"PRIu32
", B: %"PRIu32
", Unknown: %"PRIu32
"\n",
1442 picture_stats
[HEVC_PICTURE_TYPE_IDR
], picture_stats
[HEVC_PICTURE_TYPE_CRA
],
1443 picture_stats
[HEVC_PICTURE_TYPE_BLA
], picture_stats
[HEVC_PICTURE_TYPE_I
],
1444 picture_stats
[HEVC_PICTURE_TYPE_I_P
], picture_stats
[HEVC_PICTURE_TYPE_I_P_B
],
1445 picture_stats
[HEVC_PICTURE_TYPE_NONE
]);
1446 /* Copy and append the last Codec Specific info. */
1447 if( (err
= hevc_store_codec_specific( hevc_imp
, &info
->hvcC_param
)) < 0 )
1449 /* Set up the first summary. */
1450 lsmash_video_summary_t
*summary
= hevc_setup_first_summary( importer
);
1453 err
= LSMASH_ERR_NAMELESS
;
1457 lsmash_media_ts_t
*timestamp
= lsmash_malloc( num_access_units
* sizeof(lsmash_media_ts_t
) );
1460 /* Count leading samples that are undecodable. */
1461 for( uint32_t i
= 0; i
< num_access_units
; i
++ )
1463 if( npt
[i
].poc
== 0 )
1465 ++ hevc_imp
->num_undecodable
;
1467 /* Deduplicate POCs. */
1468 uint32_t max_composition_delay
= 0;
1469 nalu_deduplicate_poc( npt
, &max_composition_delay
, num_access_units
, 15 );
1470 /* Generate timestamps. */
1471 nalu_generate_timestamps_from_poc( importer
, timestamp
, npt
,
1472 &hevc_imp
->composition_reordering_present
,
1473 &hevc_imp
->last_delta
,
1474 max_composition_delay
, num_access_units
);
1475 summary
->timescale
*= 2; /* We assume that picture timing is in field level.
1476 * For HEVC, it seems time_scale is set in frame level basically.
1477 * So multiply by 2 for reducing timebase and timescale. */
1478 nalu_reduce_timescale( timestamp
, npt
, &hevc_imp
->last_delta
, &summary
->timescale
, num_access_units
);
1480 hevc_imp
->ts_list
.sample_count
= num_access_units
;
1481 hevc_imp
->ts_list
.timestamp
= timestamp
;
1484 lsmash_log_refresh_line( &logger
);
1489 static int hevc_importer_probe( importer_t
*importer
)
1491 /* Find the first start code. */
1492 hevc_importer_t
*hevc_imp
= create_hevc_importer( importer
);
1494 return LSMASH_ERR_MEMORY_ALLOC
;
1495 lsmash_bs_t
*bs
= importer
->bs
;
1496 uint64_t first_sc_head_pos
= nalu_find_first_start_code( bs
);
1498 if( first_sc_head_pos
== NALU_NO_START_CODE_FOUND
)
1500 err
= LSMASH_ERR_INVALID_DATA
;
1503 else if( first_sc_head_pos
== NALU_IO_ERROR
)
1505 err
= LSMASH_ERR_IO
;
1508 /* OK. It seems the stream has a long start code of HEVC. */
1509 importer
->info
= hevc_imp
;
1510 hevc_info_t
*info
= &hevc_imp
->info
;
1511 lsmash_bs_read_seek( bs
, first_sc_head_pos
, SEEK_SET
);
1512 hevc_imp
->sc_head_pos
= first_sc_head_pos
;
1513 if( (err
= hevc_analyze_whole_stream( importer
)) < 0 )
1515 /* Go back to the start code of the first NALU. */
1516 importer
->status
= IMPORTER_OK
;
1517 lsmash_bs_read_seek( bs
, first_sc_head_pos
, SEEK_SET
);
1518 hevc_imp
->sc_head_pos
= first_sc_head_pos
;
1519 info
->prev_nalu_type
= HEVC_NALU_TYPE_UNKNOWN
;
1520 uint8_t *temp_au
= info
->au
.data
;
1521 uint8_t *temp_incomplete_au
= info
->au
.incomplete_data
;
1522 memset( &info
->au
, 0, sizeof(hevc_access_unit_t
) );
1523 info
->au
.data
= temp_au
;
1524 info
->au
.incomplete_data
= temp_incomplete_au
;
1525 memset( &info
->slice
, 0, sizeof(hevc_slice_info_t
) );
1526 memset( &info
->vps
, 0, sizeof(hevc_vps_t
) );
1527 memset( &info
->sps
, 0, sizeof(hevc_sps_t
) );
1528 memset( &info
->pps
, 0, SIZEOF_PPS_EXCLUDING_HEAP
);
1529 for( int i
= 0; i
< HEVC_DCR_NALU_TYPE_NUM
; i
++ )
1530 lsmash_remove_entries( info
->hvcC_param
.parameter_arrays
->ps_array
[i
].list
, isom_remove_dcr_ps
);
1531 lsmash_destroy_hevc_parameter_arrays( &info
->hvcC_param_next
);
1534 remove_hevc_importer( hevc_imp
);
1535 importer
->info
= NULL
;
1536 lsmash_remove_entries( importer
->summaries
, lsmash_cleanup_summary
);
1540 static uint32_t hevc_importer_get_last_delta( importer_t
*importer
, uint32_t track_number
)
1542 debug_if( !importer
|| !importer
->info
)
1544 hevc_importer_t
*hevc_imp
= (hevc_importer_t
*)importer
->info
;
1545 if( !hevc_imp
|| track_number
!= 1 || importer
->status
!= IMPORTER_EOF
)
1547 return hevc_imp
->ts_list
.sample_count
1548 ? hevc_imp
->last_delta
1549 : UINT32_MAX
; /* arbitrary */
1552 const importer_functions hevc_importer
=
1554 { "HEVC", offsetof( importer_t
, log_level
) },
1556 hevc_importer_probe
,
1557 hevc_importer_get_accessunit
,
1558 hevc_importer_get_last_delta
,
1559 hevc_importer_cleanup