1 /* $Id: sync.c,v 1.38 2005/04/14 21:57:58 titer Exp $
3 This file is part of the HandBrake source code.
4 Homepage: <http://handbrake.m0k.org/>.
5 It may be used under the terms of the GNU General Public License. */
9 #include "samplerate.h"
10 #include "ffmpeg/avcodec.h"
13 #undef INT64_MIN /* Because it isn't defined correctly in Zeta */
15 #define INT64_MIN (-9223372036854775807LL-1)
17 #define AC3_SAMPLES_PER_FRAME 1536
34 struct hb_work_private_s
40 hb_subtitle_t
* subtitle
;
42 int64_t pts_offset_old
;
44 int64_t count_frames_max
;
45 hb_buffer_t
* cur
; /* The next picture to process */
48 hb_sync_audio_t sync_audio
[8];
51 uint64_t st_counts
[4];
56 /***********************************************************************
58 **********************************************************************/
59 static void InitAudio( hb_work_object_t
* w
, int i
);
60 static int SyncVideo( hb_work_object_t
* w
);
61 static void SyncAudio( hb_work_object_t
* w
, int i
);
62 static int NeedSilence( hb_work_object_t
* w
, hb_audio_t
* );
63 static void InsertSilence( hb_work_object_t
* w
, int i
);
64 static void UpdateState( hb_work_object_t
* w
);
66 /***********************************************************************
68 ***********************************************************************
69 * Initialize the work object
70 **********************************************************************/
71 int syncInit( hb_work_object_t
* w
, hb_job_t
* job
)
73 hb_title_t
* title
= job
->title
;
74 hb_chapter_t
* chapter
;
77 hb_work_private_t
* pv
;
79 pv
= calloc( 1, sizeof( hb_work_private_t
) );
83 pv
->pts_offset
= INT64_MIN
;
84 pv
->pts_offset_old
= INT64_MIN
;
87 /* Calculate how many video frames we are expecting */
89 for( i
= job
->chapter_start
; i
<= job
->chapter_end
; i
++ )
91 chapter
= hb_list_item( title
->list_chapter
, i
- 1 );
92 duration
+= chapter
->duration
;
95 /* 1 second safety so we're sure we won't miss anything */
96 pv
->count_frames_max
= duration
* job
->vrate
/ job
->vrate_base
/ 90000;
98 hb_log( "sync: expecting %lld video frames", pv
->count_frames_max
);
100 /* Initialize libsamplerate for every audio track we have */
101 for( i
= 0; i
< hb_list_count( title
->list_audio
); i
++ )
106 /* Get subtitle info, if any */
107 pv
->subtitle
= hb_list_item( title
->list_subtitle
, 0 );
112 /***********************************************************************
114 ***********************************************************************
116 **********************************************************************/
117 void syncClose( hb_work_object_t
* w
)
119 hb_work_private_t
* pv
= w
->private_data
;
120 hb_job_t
* job
= pv
->job
;
121 hb_title_t
* title
= job
->title
;
125 if( pv
->cur
) hb_buffer_close( &pv
->cur
);
127 for( i
= 0; i
< hb_list_count( title
->list_audio
); i
++ )
129 if( job
->acodec
& HB_ACODEC_AC3
)
131 free( pv
->sync_audio
[i
].ac3_buf
);
135 src_delete( pv
->sync_audio
[i
].state
);
140 w
->private_data
= NULL
;
143 /***********************************************************************
145 ***********************************************************************
146 * The root routine of this work abject
147 **********************************************************************/
148 int syncWork( hb_work_object_t
* w
, hb_buffer_t
** unused1
,
149 hb_buffer_t
** unused2
)
151 hb_work_private_t
* pv
= w
->private_data
;
154 /* If we ever got a video frame, handle audio now */
155 if( pv
->pts_offset
!= INT64_MIN
)
157 for( i
= 0; i
< hb_list_count( pv
->job
->title
->list_audio
); i
++ )
164 return SyncVideo( w
);
167 hb_work_object_t hb_sync
=
176 static void InitAudio( hb_work_object_t
* w
, int i
)
178 hb_work_private_t
* pv
= w
->private_data
;
179 hb_job_t
* job
= pv
->job
;
180 hb_title_t
* title
= job
->title
;
181 hb_sync_audio_t
* sync
;
183 sync
= &pv
->sync_audio
[i
];
184 sync
->audio
= hb_list_item( title
->list_audio
, i
);
186 if( job
->acodec
& HB_ACODEC_AC3
)
188 /* Have a silent AC-3 frame ready in case we have to fill a
194 codec
= avcodec_find_encoder( CODEC_ID_AC3
);
195 c
= avcodec_alloc_context();
197 c
->bit_rate
= sync
->audio
->bitrate
;
198 c
->sample_rate
= sync
->audio
->rate
;
201 if( avcodec_open( c
, codec
) < 0 )
203 hb_log( "sync: avcodec_open failed" );
207 zeros
= calloc( AC3_SAMPLES_PER_FRAME
*
208 sizeof( short ) * c
->channels
, 1 );
209 sync
->ac3_size
= sync
->audio
->bitrate
* AC3_SAMPLES_PER_FRAME
/
210 sync
->audio
->rate
/ 8;
211 sync
->ac3_buf
= malloc( sync
->ac3_size
);
213 if( avcodec_encode_audio( c
, sync
->ac3_buf
, sync
->ac3_size
,
214 zeros
) != sync
->ac3_size
)
216 hb_log( "sync: avcodec_encode_audio failed" );
225 /* Initialize libsamplerate */
227 sync
->state
= src_new( SRC_LINEAR
, HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(sync
->audio
->amixdown
), &error
);
228 sync
->data
.end_of_input
= 0;
234 #define PTS_DISCONTINUITY_TOLERANCE 90000
236 /***********************************************************************
238 ***********************************************************************
240 **********************************************************************/
241 static int SyncVideo( hb_work_object_t
* w
)
243 hb_work_private_t
* pv
= w
->private_data
;
244 hb_buffer_t
* cur
, * next
, * sub
= NULL
;
245 hb_job_t
* job
= pv
->job
;
246 int64_t pts_expected
;
254 if( hb_thread_has_exited( job
->reader
) &&
255 !hb_fifo_size( job
->fifo_mpeg2
) &&
256 !hb_fifo_size( job
->fifo_raw
) )
258 /* All video data has been processed already, we won't get
260 hb_log( "sync: got %lld frames, %lld expected",
261 pv
->count_frames
, pv
->count_frames_max
);
264 hb_buffer_t
* buf_tmp
;
266 // Drop an empty buffer into our output to ensure that things
267 // get flushed all the way out.
268 buf_tmp
= hb_buffer_init(0); // Empty end buffer
269 hb_fifo_push( job
->fifo_sync
, buf_tmp
);
274 if( !pv
->cur
&& !( pv
->cur
= hb_fifo_get( job
->fifo_raw
) ) )
276 /* We haven't even got a frame yet */
281 /* At this point we have a frame to process. Let's check
282 1) if we will be able to push into the fifo ahead
283 2) if the next frame is there already, since we need it to
284 know whether we'll have to repeat the current frame or not */
285 while( !hb_fifo_is_full( job
->fifo_sync
) &&
286 ( next
= hb_fifo_see( job
->fifo_raw
) ) )
288 hb_buffer_t
* buf_tmp
;
290 if( pv
->pts_offset
== INT64_MIN
)
292 /* This is our first frame */
293 hb_log( "sync: first pts is %lld", cur
->start
);
294 pv
->pts_offset
= cur
->start
;
297 /* Check for PTS jumps over 0.5 second */
298 if( next
->start
< cur
->start
- PTS_DISCONTINUITY_TOLERANCE
||
299 next
->start
> cur
->start
+ PTS_DISCONTINUITY_TOLERANCE
)
301 hb_log( "Sync: Video PTS discontinuity (current buffer start=%lld, next buffer start=%lld), trash Video",
302 cur
->start
, next
->start
);
304 /* Trash all subtitles */
307 while( ( sub
= hb_fifo_get( pv
->subtitle
->fifo_raw
) ) )
309 hb_buffer_close( &sub
);
313 /* Trash current picture */
314 /* Also, make sure we don't trash a chapter break */
315 chap_break
= cur
->new_chap
;
316 hb_buffer_close( &cur
);
317 pv
->cur
= cur
= hb_fifo_get( job
->fifo_raw
);
318 cur
->new_chap
|= chap_break
; // Don't stomp existing chapter breaks
320 /* Calculate new offset */
321 pv
->pts_offset_old
= pv
->pts_offset
;
322 pv
->pts_offset
= cur
->start
-
323 pv
->count_frames
* pv
->job
->vrate_base
/ 300;
327 /* Look for a subtitle for this frame */
331 while( ( sub
= hb_fifo_see( pv
->subtitle
->fifo_raw
) ) )
333 /* If two subtitles overlap, make the first one stop
334 when the second one starts */
335 sub2
= hb_fifo_see2( pv
->subtitle
->fifo_raw
);
336 if( sub2
&& sub
->stop
> sub2
->start
)
337 sub
->stop
= sub2
->start
;
339 if( sub
->stop
> cur
->start
)
342 /* The subtitle is older than this picture, trash it */
343 sub
= hb_fifo_get( pv
->subtitle
->fifo_raw
);
344 hb_buffer_close( &sub
);
347 /* If we have subtitles left in the fifo, check if we should
348 apply the first one to the current frame or if we should
350 if( sub
&& sub
->start
> cur
->start
)
356 /* The PTS of the frame we are expecting now */
357 pts_expected
= pv
->pts_offset
+
358 pv
->count_frames
* pv
->job
->vrate_base
/ 300;
360 if( cur
->start
< pts_expected
- pv
->job
->vrate_base
/ 300 / 2 &&
361 next
->start
< pts_expected
+ pv
->job
->vrate_base
/ 300 / 2 )
363 /* The current frame is too old but the next one matches,
365 /* Also, make sure we don't trash a chapter break */
366 chap_break
= cur
->new_chap
;
367 hb_buffer_close( &cur
);
368 pv
->cur
= cur
= hb_fifo_get( job
->fifo_raw
);
369 cur
->new_chap
|= chap_break
; // Make sure we don't stomp the existing one.
374 if( next
->start
> pts_expected
+ 3 * pv
->job
->vrate_base
/ 300 / 2 )
376 /* We'll need the current frame more than one time. Make a
377 copy of it and keep it */
378 buf_tmp
= hb_buffer_init( cur
->size
);
379 memcpy( buf_tmp
->data
, cur
->data
, cur
->size
);
383 /* The frame has the expected date and won't have to be
384 duplicated, just put it through */
386 pv
->cur
= cur
= hb_fifo_get( job
->fifo_raw
);
389 /* Replace those MPEG-2 dates with our dates */
390 buf_tmp
->start
= (uint64_t) pv
->count_frames
*
391 pv
->job
->vrate_base
/ 300;
392 buf_tmp
->stop
= (uint64_t) ( pv
->count_frames
+ 1 ) *
393 pv
->job
->vrate_base
/ 300;
395 /* If we have a subtitle for this picture, copy it */
396 /* FIXME: we should avoid this memcpy */
399 buf_tmp
->sub
= hb_buffer_init( sub
->size
);
400 buf_tmp
->sub
->x
= sub
->x
;
401 buf_tmp
->sub
->y
= sub
->y
;
402 buf_tmp
->sub
->width
= sub
->width
;
403 buf_tmp
->sub
->height
= sub
->height
;
404 memcpy( buf_tmp
->sub
->data
, sub
->data
, sub
->size
);
407 /* Push the frame to the renderer */
408 hb_fifo_push( job
->fifo_sync
, buf_tmp
);
413 /* Make sure we won't get more frames then expected */
414 if( pv
->count_frames
>= pv
->count_frames_max
)
416 hb_log( "sync: got %lld frames", pv
->count_frames
);
419 // Drop an empty buffer into our output to ensure that things
420 // get flushed all the way out.
421 buf_tmp
= hb_buffer_init(0); // Empty end buffer
422 hb_fifo_push( job
->fifo_sync
, buf_tmp
);
431 /***********************************************************************
433 ***********************************************************************
435 **********************************************************************/
436 static void SyncAudio( hb_work_object_t
* w
, int i
)
438 hb_work_private_t
* pv
= w
->private_data
;
442 hb_sync_audio_t
* sync
;
447 int64_t pts_expected
;
451 sync
= &pv
->sync_audio
[i
];
454 if( job
->acodec
& HB_ACODEC_AC3
)
456 fifo
= audio
->fifo_out
;
461 fifo
= audio
->fifo_sync
;
465 while( !hb_fifo_is_full( fifo
) &&
466 ( buf
= hb_fifo_see( audio
->fifo_raw
) ) )
468 /* The PTS of the samples we are expecting now */
469 pts_expected
= pv
->pts_offset
+ sync
->count_frames
* 90000 / rate
;
472 * Using the same logic as the Video have we crossed a VOB boundary as detected
473 * by the expected PTS and the PTS of our audio being out by more than the tolerance
476 if( ( buf
->start
> pts_expected
+ PTS_DISCONTINUITY_TOLERANCE
||
477 buf
->start
< pts_expected
- PTS_DISCONTINUITY_TOLERANCE
) &&
478 pv
->pts_offset_old
> INT64_MIN
)
481 * Useful debug, but too verbose for normal use.
482 hb_log("Sync: Audio discontinuity (%lld < %lld < %lld)",
483 pts_expected - PTS_DISCONTINUITY_TOLERANCE, buf->start,
484 pts_expected + PTS_DISCONTINUITY_TOLERANCE );
487 /* There has been a PTS discontinuity, and this frame might
488 be from before the discontinuity*/
489 pts_expected
= pv
->pts_offset_old
+ sync
->count_frames
*
493 * Is the audio from a valid period given the previous Video PTS. I.e. has there
494 * just been a video PTS discontinuity and this audio belongs to the vdeo from
497 if( buf
->start
> pts_expected
+ PTS_DISCONTINUITY_TOLERANCE
||
498 buf
->start
< pts_expected
- PTS_DISCONTINUITY_TOLERANCE
)
501 * It's outside of our tolerance for where the video is now, and it's outside
502 * of the tolerance for where we have been in the case of a VOB change.
503 * Try and reconverge regardless. so continue on to our convergence
504 * code below which will kick in as it will be more than 100ms out.
506 * Note that trashing the Audio could make things worse if the Audio is in
507 * front because we will end up diverging even more. We need to hold on
508 * to the audio until the video catches up.
510 hb_log("Sync: Audio is way out of sync, attempt to reconverge from current video PTS");
513 * It wasn't from the old place, so we must be from the new, but just too far out. So attempt to
514 * reconverge by resetting the point we want to be to where we are currently wanting to be.
516 pts_expected
= pv
->pts_offset
+ sync
->count_frames
* 90000 / rate
;
518 /* Use the older offset */
519 start
= pts_expected
- pv
->pts_offset_old
;
524 start
= pts_expected
- pv
->pts_offset
;
527 /* Tolerance: 100 ms */
528 if( buf
->start
< pts_expected
- 9000 )
530 /* Audio is behind the Video, trash it, can't use it now. */
531 hb_log( "Sync: Audio PTS (%lld) < Video PTS (%lld) by greater than 100ms, trashing audio to reconverge",
532 buf
->start
, pts_expected
);
533 buf
= hb_fifo_get( audio
->fifo_raw
);
534 hb_buffer_close( &buf
);
537 else if( buf
->start
> pts_expected
+ 9000 )
539 /* Audio is ahead of the Video, insert silence until we catch up*/
540 hb_log("Sync: Audio PTS (%lld) > Video PTS (%lld) by greater than 100ms insert silence until reconverged", buf
->start
, pts_expected
);
541 InsertSilence( w
, i
);
545 if( job
->acodec
& HB_ACODEC_AC3
)
547 buf
= hb_fifo_get( audio
->fifo_raw
);
549 buf
->stop
= start
+ 90000 * AC3_SAMPLES_PER_FRAME
/ rate
;
551 sync
->count_frames
+= AC3_SAMPLES_PER_FRAME
;
555 hb_buffer_t
* buf_raw
= hb_fifo_get( audio
->fifo_raw
);
557 int count_in
, count_out
;
559 count_in
= buf_raw
->size
/ HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio
->amixdown
) / sizeof( float );
560 count_out
= ( buf_raw
->stop
- buf_raw
->start
) * job
->arate
/ 90000;
561 if( buf
->start
< pts_expected
- 1500 )
563 else if( buf
->start
> pts_expected
+ 1500 )
566 sync
->data
.data_in
= (float *) buf_raw
->data
;
567 sync
->data
.input_frames
= count_in
;
568 sync
->data
.output_frames
= count_out
;
570 sync
->data
.src_ratio
= (double) sync
->data
.output_frames
/
571 (double) sync
->data
.input_frames
;
573 buf
= hb_buffer_init( sync
->data
.output_frames
* HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio
->amixdown
) *
575 sync
->data
.data_out
= (float *) buf
->data
;
576 if( src_process( sync
->state
, &sync
->data
) )
578 /* XXX If this happens, we're screwed */
579 hb_log( "sync: src_process failed" );
581 hb_buffer_close( &buf_raw
);
583 buf
->size
= sync
->data
.output_frames_gen
* HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(audio
->amixdown
) * sizeof( float );
585 /* Set dates for resampled data */
587 buf
->stop
= start
+ sync
->data
.output_frames_gen
*
590 sync
->count_frames
+= sync
->data
.output_frames_gen
;
593 buf
->frametype
= HB_FRAME_AUDIO
;
594 hb_fifo_push( fifo
, buf
);
597 if( NeedSilence( w
, audio
) )
599 InsertSilence( w
, i
);
603 static int NeedSilence( hb_work_object_t
* w
, hb_audio_t
* audio
)
605 hb_work_private_t
* pv
= w
->private_data
;
606 hb_job_t
* job
= pv
->job
;
608 if( hb_fifo_size( audio
->fifo_in
) ||
609 hb_fifo_size( audio
->fifo_raw
) ||
610 hb_fifo_size( audio
->fifo_sync
) ||
611 hb_fifo_size( audio
->fifo_out
) )
613 /* We have some audio, we are fine */
617 /* No audio left in fifos */
619 if( hb_thread_has_exited( job
->reader
) )
621 /* We might miss some audio to complete encoding and muxing
623 hb_log("Reader has exited early, inserting silence.");
627 if( hb_fifo_is_full( job
->fifo_mpeg2
) &&
628 hb_fifo_is_full( job
->fifo_raw
) &&
629 hb_fifo_is_full( job
->fifo_sync
) &&
630 hb_fifo_is_full( job
->fifo_render
) &&
631 hb_fifo_is_full( job
->fifo_mpeg4
) )
633 /* Too much video and no audio, oh-oh */
634 hb_log("Still got some video - and nothing in the audio fifo, insert silence");
641 static void InsertSilence( hb_work_object_t
* w
, int i
)
643 hb_work_private_t
* pv
= w
->private_data
;
645 hb_sync_audio_t
* sync
;
649 sync
= &pv
->sync_audio
[i
];
651 if( job
->acodec
& HB_ACODEC_AC3
)
653 buf
= hb_buffer_init( sync
->ac3_size
);
654 buf
->start
= sync
->count_frames
* 90000 / sync
->audio
->rate
;
655 buf
->stop
= buf
->start
+ 90000 * AC3_SAMPLES_PER_FRAME
/
657 memcpy( buf
->data
, sync
->ac3_buf
, buf
->size
);
659 hb_log( "sync: adding a silent AC-3 frame for track %x",
661 hb_fifo_push( sync
->audio
->fifo_out
, buf
);
663 sync
->count_frames
+= AC3_SAMPLES_PER_FRAME
;
668 buf
= hb_buffer_init( HB_AMIXDOWN_GET_DISCRETE_CHANNEL_COUNT(sync
->audio
->amixdown
) * job
->arate
/ 20 *
670 buf
->start
= sync
->count_frames
* 90000 / job
->arate
;
671 buf
->stop
= buf
->start
+ 90000 / 20;
672 memset( buf
->data
, 0, buf
->size
);
674 hb_log( "sync: adding 50 ms of silence for track %x",
676 hb_fifo_push( sync
->audio
->fifo_sync
, buf
);
678 sync
->count_frames
+= job
->arate
/ 20;
682 static void UpdateState( hb_work_object_t
* w
)
684 hb_work_private_t
* pv
= w
->private_data
;
687 if( !pv
->count_frames
)
689 pv
->st_first
= hb_get_date();
693 if( hb_get_date() > pv
->st_dates
[3] + 1000 )
695 memmove( &pv
->st_dates
[0], &pv
->st_dates
[1],
696 3 * sizeof( uint64_t ) );
697 memmove( &pv
->st_counts
[0], &pv
->st_counts
[1],
698 3 * sizeof( uint64_t ) );
699 pv
->st_dates
[3] = hb_get_date();
700 pv
->st_counts
[3] = pv
->count_frames
;
703 #define p state.param.working
704 state
.state
= HB_STATE_WORKING
;
705 p
.progress
= (float) pv
->count_frames
/ (float) pv
->count_frames_max
;
706 if( p
.progress
> 1.0 )
710 p
.rate_cur
= 1000.0 *
711 (float) ( pv
->st_counts
[3] - pv
->st_counts
[0] ) /
712 (float) ( pv
->st_dates
[3] - pv
->st_dates
[0] );
713 if( hb_get_date() > pv
->st_first
+ 4000 )
716 p
.rate_avg
= 1000.0 * (float) pv
->st_counts
[3] /
717 (float) ( pv
->st_dates
[3] - pv
->st_first
);
718 eta
= (float) ( pv
->count_frames_max
- pv
->st_counts
[3] ) /
720 p
.hours
= eta
/ 3600;
721 p
.minutes
= ( eta
% 3600 ) / 60;
722 p
.seconds
= eta
% 60;
733 hb_set_state( pv
->job
->h
, &state
);