From 2b7347be805d17a2b6f06e276e897c51431b9c6c Mon Sep 17 00:00:00 2001 From: Sylvain BERTRAND Date: Tue, 22 Sep 2020 17:38:46 +0000 Subject: [PATCH] npv: fix filter time base handling, prepare discontinuity support --- npa/TODO | 1 + npv/audio/filt/local/code.frag.c | 18 +++++---- npv/audio/filt/local/state.frag.c | 1 - npv/audio/filt/namespace/main.c | 2 - npv/audio/filt/public.h | 4 +- npv/audio/filt/public/code.frag.c | 9 +++-- npv/audio/filt/public/state.frag.h | 2 + npv/audio/local/code.frag.c | 3 +- npv/clk/local/state.frag.c | 2 +- npv/clk/main.c | 1 + npv/clk/public.h | 4 +- npv/clk/public/code.frag.c | 49 ++++++++++------------- npv/local/code.frag.c | 41 +++++++++---------- npv/main.c | 1 + npv/namespace/ffmpeg.h | 2 + npv/pipeline/local/code.frag.c | 82 ++++++++++++++++++-------------------- npv/pipeline/namespace/main.c | 6 ++- npv/video/local/code.frag.c | 9 +++-- 18 files changed, 121 insertions(+), 116 deletions(-) diff --git a/npa/TODO b/npa/TODO index 35ff3e5..70b2a25 100644 --- a/npa/TODO +++ b/npa/TODO @@ -1,4 +1,5 @@ not ordered: +- port npv proper usage of filter time base - port npv dynamic audio reconfiguration - port npv initial ffmpeg audio parameter override code - we were lied to: presentation/decoding timestamps from a demuxer can be diff --git a/npv/audio/filt/local/code.frag.c b/npv/audio/filt/local/code.frag.c index 432cf92..7ce8f1f 100644 --- a/npv/audio/filt/local/code.frag.c +++ b/npv/audio/filt/local/code.frag.c @@ -34,11 +34,11 @@ STATIC bool is_recfg_required(avutil_audio_set_ref_t *src_set) return true; return false; } -STATIC void abufsrc_cfg(int chans_n, uint64_t chans_layout, int rate, - enum avutil_audio_fr_fmt_t fmt, bool print_info) +STATIC void abufsrc_cfg(avutil_rational_t tb, int chans_n, + uint64_t chans_layout, int rate, enum avutil_audio_fr_fmt_t fmt, + bool print_info) { int r; - avutil_rational_t time_base; u8 chans_layout_str[STR_SZ]; /* should be overkill */ abufsrc_l.this = avfilter_get_by_name("abuffer"); @@ -65,6 +65,10 @@ STATIC void abufsrc_cfg(int chans_n, uint64_t chans_layout, int rate, AVUTIL_OPT_SEARCH_CHILDREN); if (r < 0) fatal("audio buffer source context:unable to set the decoder channel layout option\n"); + r = avutil_opt_set_q(abufsrc_l.ctx, "time_base", tb, + AVUTIL_OPT_SEARCH_CHILDREN); + if (r < 0) + fatal("audio buffer source context:unable to set the time base option\n"); r = avfilter_init_str(abufsrc_l.ctx, 0); if (r < 0) fatal("audio buffer source context:unable to initialize\n"); @@ -136,11 +140,11 @@ STATIC void abufsink_cfg(void) abufsink_l = avfilter_get_by_name("abuffersink"); if (abufsink_l == 0) fatal("audio buffer sink:could not find the filter\n"); - abufsink_ctx_l = avfilter_graph_alloc_filt(graph_l, abufsink_l, + filt_p.abufsink_ctx = avfilter_graph_alloc_filt(graph_l, abufsink_l, "sink_abuf"); - if (abufsink_ctx_l == 0) + if (filt_p.abufsink_ctx == 0) fatal("audio buffer sink context:could not allocate the instance in the filter graph\n"); - r = avfilter_init_str(abufsink_ctx_l, 0); + r = avfilter_init_str(filt_p.abufsink_ctx, 0); if (r < 0) fatal("audio buffer sink context:unable to initialize\n"); } @@ -152,7 +156,6 @@ STATIC void init_once_local(void) vol_l = 0; afmt_ctx_l = 0; afmt_l = 0; - abufsink_ctx_l = 0; abufsink_l = 0; /* floating point strs are localized... erk... */ snprintf(double_zero_l10n_str_l, sizeof(double_zero_l10n_str_l), @@ -166,4 +169,5 @@ STATIC void init_once_public(double initial_vol) filt_p.pcm_written_ufrs_n = 0; filt_p.vol = initial_vol; filt_p.muted = false; + filt_p.abufsink_ctx = 0; } diff --git a/npv/audio/filt/local/state.frag.c b/npv/audio/filt/local/state.frag.c index a2b7830..5e2e6d9 100644 --- a/npv/audio/filt/local/state.frag.c +++ b/npv/audio/filt/local/state.frag.c @@ -15,5 +15,4 @@ STATIC const avfilter_filt_t *vol_l; STATIC u8 double_zero_l10n_str_l[sizeof("xxx.xx")]; STATIC avfilter_filt_ctx_t *afmt_ctx_l; STATIC const avfilter_filt_t *afmt_l; -STATIC avfilter_filt_ctx_t *abufsink_ctx_l; STATIC const avfilter_filt_t *abufsink_l; diff --git a/npv/audio/filt/namespace/main.c b/npv/audio/filt/namespace/main.c index 83e3c2d..476c07a 100644 --- a/npv/audio/filt/namespace/main.c +++ b/npv/audio/filt/namespace/main.c @@ -1,6 +1,5 @@ #ifndef CLEANUP #define abufsink_cfg npv_audio_filt_abufsink_cfg -#define abufsink_ctx_l npv_audio_filt_abufsink_ctx_l #define abufsink_l npv_audio_filt_abufsink_l #define abufsrc_cfg npv_audio_filt_abufsrc_cfg #define abufsrc_l npv_audio_filt_abufsrc_l @@ -27,7 +26,6 @@ /*============================================================================*/ #else #undef abufsink_cfg -#undef abufsink_ctx_l #undef abufsink_l #undef abufsrc_cfg #undef abufsrc_l diff --git a/npv/audio/filt/public.h b/npv/audio/filt/public.h index 9ff44ce..2e06c71 100644 --- a/npv/audio/filt/public.h +++ b/npv/audio/filt/public.h @@ -7,6 +7,7 @@ #include #include #include +#include #include #include "npv/c_fixing.h" #include "npv/global.h" @@ -19,7 +20,8 @@ #include "npv/audio/filt/public/state.frag.h" /*---------------------------------------------------------------------------*/ STATIC void init_once(double initial_vol); -STATIC void cfg(int src_chans_n, uint64_t src_chans_layout, int src_rate, +STATIC void cfg(avutil_rational_t tb, + int src_chans_n, uint64_t src_chans_layout, int src_rate, enum avutil_audio_fr_fmt_t src_fmt, bool muted, double vol, int dst_chans_n, uint64_t dst_chans_layout, int dst_rate, diff --git a/npv/audio/filt/public/code.frag.c b/npv/audio/filt/public/code.frag.c index 77bd9c9..7cb3f17 100644 --- a/npv/audio/filt/public/code.frag.c +++ b/npv/audio/filt/public/code.frag.c @@ -81,7 +81,7 @@ STATIC u8 filt_set_get(void) * the last dec set should switch the filt in draining mode, and * filt_p.set won't matter. */ - r = avfilter_bufsink_get_audio_set(abufsink_ctx_l, filt_p.set); + r = avfilter_bufsink_get_audio_set(filt_p.abufsink_ctx, filt_p.set); if (r >= 0) { filt_p.pcm_written_ufrs_n = 0; return HAVE_FILT_SET; @@ -104,7 +104,8 @@ STATIC void init_once(double initial_vol) init_once_local(); init_once_public(initial_vol); } -STATIC void cfg(int src_chans_n, uint64_t src_chans_layout, int src_rate, +STATIC void cfg(avutil_rational_t tb, + int src_chans_n, uint64_t src_chans_layout, int src_rate, enum avutil_audio_fr_fmt_t src_fmt, bool muted, double vol, int dst_chans_n, uint64_t dst_chans_layout, int dst_rate, @@ -119,7 +120,7 @@ STATIC void cfg(int src_chans_n, uint64_t src_chans_layout, int src_rate, graph_l = avfilter_graph_alloc(); if (graph_l == 0) fatal("unable to create filter graph\n"); - abufsrc_cfg(src_chans_n, src_chans_layout, src_rate, src_fmt, + abufsrc_cfg(tb, src_chans_n, src_chans_layout, src_rate, src_fmt, print_info); /*--------------------------------------------------------------------*/ abufsrc_l.key.chans_n = src_chans_n; @@ -136,7 +137,7 @@ STATIC void cfg(int src_chans_n, uint64_t src_chans_layout, int src_rate, r = avfilter_link(vol_ctx_l, 0, afmt_ctx_l, 0); if (r < 0) fatal("unable to connect the volume filter to the audio format filter\n"); - r = avfilter_link(afmt_ctx_l, 0, abufsink_ctx_l, 0); + r = avfilter_link(afmt_ctx_l, 0, filt_p.abufsink_ctx, 0); if (r < 0) fatal("unable to connect the audio format filter to the audio buffer sink filter\n"); r = avfilter_graph_config(graph_l, 0); diff --git a/npv/audio/filt/public/state.frag.h b/npv/audio/filt/public/state.frag.h index 1144830..2e885e0 100644 --- a/npv/audio/filt/public/state.frag.h +++ b/npv/audio/filt/public/state.frag.h @@ -9,4 +9,6 @@ STATIC struct { double vol; bool muted; + /* we will need query some output props from the filte */ + avfilter_filt_ctx_t *abufsink_ctx; } filt_p; diff --git a/npv/audio/local/code.frag.c b/npv/audio/local/code.frag.c index 3f827fe..e10e8bd 100644 --- a/npv/audio/local/code.frag.c +++ b/npv/audio/local/code.frag.c @@ -686,7 +686,8 @@ STATIC u8 dec_set_filter(void) /* FILT_RECFG_REQUIRED */ pcm2ff_strict(npv_audio_pcm_p, &dst_chans_n, &dst_chans_layout, &dst_rate, &dst_fmt, PRINT_INFO); - filt_cfg(new_chans_n, new_chans_layout, new_rate, new_fmt, + filt_cfg(st_p.tb, + new_chans_n, new_chans_layout, new_rate, new_fmt, filt_p.muted, filt_p.vol, dst_chans_n, dst_chans_layout, dst_rate, dst_fmt, PRINT_INFO); diff --git a/npv/clk/local/state.frag.c b/npv/clk/local/state.frag.c index 209a0de..e125745 100644 --- a/npv/clk/local/state.frag.c +++ b/npv/clk/local/state.frag.c @@ -24,7 +24,7 @@ STATIC struct { */ snd_pcm_ufrs_t written_ufrs_n; - s64 audio_st_ts; /* ff audio st tb units */ + s64 audio_filt_ts; /* in ff audio filt tb units */ } ref; struct { snd_pcm_status_t *status; diff --git a/npv/clk/main.c b/npv/clk/main.c index aeeccfb..edc559f 100644 --- a/npv/clk/main.c +++ b/npv/clk/main.c @@ -12,6 +12,7 @@ #include #include "npv/c_fixing.h" #include "npv/audio/public.h" +#include "npv/audio/filt/public.h" #include "npv/video/public.h" /*---------------------------------------------------------------------------*/ #include "npv/namespace/alsa.h" diff --git a/npv/clk/public.h b/npv/clk/public.h index bae0b52..cf3cfbe 100644 --- a/npv/clk/public.h +++ b/npv/clk/public.h @@ -7,9 +7,9 @@ #include "npv/namespace/alsa.h" /*---------------------------------------------------------------------------*/ STATIC void npv_clk_init_once(void); -STATIC u8 npv_clk_get_audio_st_ts(s64 *ts); +STATIC u8 npv_clk_get_audio_filt_ts(s64 *ts); STATIC u8 npv_clk_get_video_st_ts(s64 *ts); -STATIC void npv_clk_ref_time_point_update(s64 audio_ts, +STATIC void npv_clk_ref_time_point_update(s64 audio_filt_ts, snd_pcm_ufrs_t written_ufrs_n); STATIC void npv_clk_invalidate(void); STATIC void npv_clk_pause(void); diff --git a/npv/clk/public/code.frag.c b/npv/clk/public/code.frag.c index c8a4604..84aafba 100644 --- a/npv/clk/public/code.frag.c +++ b/npv/clk/public/code.frag.c @@ -15,7 +15,7 @@ STATIC void npv_clk_init_once(void) npv_clk_l.paused = false; } #define NO_REF_TIME_POINT 1 -STATIC u8 npv_clk_get_audio_st_ts(s64 *ts) +STATIC u8 npv_clk_get_audio_filt_ts(s64 *ts) { int r; snd_pcm_audio_tstamp_config_t ac; @@ -24,9 +24,10 @@ STATIC u8 npv_clk_get_audio_st_ts(s64 *ts) f64 ref_ns; f64 ref_status_ns; f64 ref_delay_frs_n; - f64 ref_audio_ts; - f64 audio_tb_num; - f64 audio_tb_den; + f64 ref_audio_filt_ts; + avutil_rational_t audio_filt_tb; + f64 audio_filt_tb_num; + f64 audio_filt_tb_den; f64 audio_rate_num; unsigned int audio_rate_num_ui; f64 audio_rate_den; @@ -53,10 +54,12 @@ STATIC u8 npv_clk_get_audio_st_ts(s64 *ts) snd_pcm_status_get_audio_htstamp(npv_clk_l.ref.status, &hts); ref_status_ns = (f64)hts.tv_sec * 1e9 + (f64)hts.tv_nsec; - ref_audio_ts = (f64)npv_clk_l.ref.audio_st_ts; + ref_audio_filt_ts = (f64)npv_clk_l.ref.audio_filt_ts; ref_delay_frs_n = (f64)snd_pcm_status_get_delay(npv_clk_l.ref.status); - audio_tb_num = (f64)npv_audio_st_p.tb.num; - audio_tb_den = (f64)npv_audio_st_p.tb.den; + /* we are writting the output of the filt to the audio pcm */ + audio_filt_tb = avfilter_bufsink_tb_get(npv_audio_filt_p.abufsink_ctx); + audio_filt_tb_num = (f64)audio_filt_tb.num; + audio_filt_tb_den = (f64)audio_filt_tb.den; /*--------------------------------------------------------------------*/ r = snd_pcm_hw_params_current(npv_audio_pcm_p, npv_clk_pcm_hw_params_l); if (r != 0) @@ -74,39 +77,27 @@ STATIC u8 npv_clk_get_audio_st_ts(s64 *ts) ref_ns = ref_status_ns + (ref_delay_frs_n - written_frs_n) * audio_rate_den * 1e9 / audio_rate_num; /* basic linear interpolation */ - now_audio_ts = audio_tb_den * 1e-9 * (now_ns - ref_ns) / audio_tb_num - + ref_audio_ts; + now_audio_ts = audio_filt_tb_den * 1e-9 * (now_ns - ref_ns) + / audio_filt_tb_num + ref_audio_filt_ts; *ts = (s64)lrint(now_audio_ts); return TS_RETURNED; } #undef NO_REF_TIME_POINT STATIC u8 npv_clk_get_video_st_ts(s64 *ts) { - f64 audio_tb_num; - f64 audio_tb_den; - f64 video_tb_num; - f64 video_tb_den; - f64 now_audio_ts; - s64 now_audio_ts_s64; - f64 now_video_ts; + avutil_rational_t audio_filt_tb; + s64 now_audio_filt_ts; u8 r; - audio_tb_num = (f64)npv_audio_st_p.tb.num; - audio_tb_den = (f64)npv_audio_st_p.tb.den; - video_tb_num = (f64)npv_video_st_p.tb.num; - video_tb_den = (f64)npv_video_st_p.tb.den; - - r = npv_clk_get_audio_st_ts(&now_audio_ts_s64); + audio_filt_tb = avfilter_bufsink_tb_get(npv_audio_filt_p.abufsink_ctx); + r = npv_clk_get_audio_filt_ts(&now_audio_filt_ts); if (r != TS_RETURNED) return r; - now_audio_ts = (f64)now_audio_ts_s64; - /* basic conversion */ - now_video_ts = audio_tb_num * video_tb_den * now_audio_ts - / (audio_tb_den * video_tb_num); - *ts = (s64)lrint(now_video_ts); + *ts = avutil_rescale_q(now_audio_filt_ts, audio_filt_tb, + npv_video_st_p.tb); return TS_RETURNED; } -STATIC void npv_clk_ref_time_point_update(s64 audio_ts, +STATIC void npv_clk_ref_time_point_update(s64 audio_filt_ts, snd_pcm_ufrs_t written_ufrs_n) { int r; @@ -119,7 +110,7 @@ STATIC void npv_clk_ref_time_point_update(s64 audio_ts, r = snd_pcm_status(npv_audio_pcm_p, npv_clk_l.ref.status); if (r < 0) npv_clk_fatal("unable to sample timing information for reference time point\n"); - npv_clk_l.ref.audio_st_ts = audio_ts; + npv_clk_l.ref.audio_filt_ts = audio_filt_ts; npv_clk_l.ref.written_ufrs_n = written_ufrs_n; npv_clk_l.ref_valid = true; } diff --git a/npv/local/code.frag.c b/npv/local/code.frag.c index 6ad6944..13f11e5 100644 --- a/npv/local/code.frag.c +++ b/npv/local/code.frag.c @@ -621,9 +621,11 @@ STATIC void seek_x(s64 delta) { int a; u8 r; - s64 new_audio_ts; - s64 new_video_ts; - s64 audio_now; + s64 now_audio_filt_ts; + avutil_rational_t audio_filt_tb; + s64 new_audio_filt_ts; + s64 new_audio_st_ts; + s64 now_audio_st_ts; if (npv_audio_draining_p) { warning("seek:audio is draining, seeking disable\n"); @@ -636,32 +638,28 @@ STATIC void seek_x(s64 delta) seek_lock(); - r = npv_clk_get_audio_st_ts(&audio_now); + r = npv_clk_get_audio_filt_ts(&now_audio_filt_ts); if (r != TS_FROM_CLK_OK) { warning("seek:audio:clock timestamp unavailable, ignoring command\n"); seek_unlock(); return; } (void)snd_pcm_drop(npv_audio_pcm_p); - /* - * XXX: a set of sts can share the same id for seeking. if they share - * the same id then the tbs should be the same. - */ + /* XXX: a set of sts can share the same id for seeking */ /*--------------------------------------------------------------------*/ - new_audio_ts = audio_now + delta * npv_audio_st_p.tb.den - / npv_audio_st_p.tb.num; - /* rewind capping if possible */ - if (npv_audio_st_p.start_time != AV_NOPTS_VALUE) - if (new_audio_ts < npv_audio_st_p.start_time) - new_audio_ts = npv_audio_st_p.start_time; - pout("trying to seek to %"PRId64" audio stream time base units\n", new_audio_ts); - a = avformat_seek_pkt(npv_fmt_ctx_p, npv_audio_st_p.id, new_audio_ts, + audio_filt_tb = avfilter_bufsink_tb_get(npv_audio_filt_p.abufsink_ctx); + new_audio_filt_ts = now_audio_filt_ts + delta * audio_filt_tb.den + / audio_filt_tb.num; + new_audio_st_ts = avutil_rescale_q(new_audio_filt_ts, audio_filt_tb, + npv_audio_st_p.tb); + pout("trying to seek to %"PRId64" audio filter time base units/%"PRId64" audio stream time base units\n", new_audio_filt_ts, new_audio_st_ts); + a = avformat_seek_pkt(npv_fmt_ctx_p, npv_audio_st_p.id, new_audio_st_ts, 0); if (a < 0) { - pout("unable to seek to %"PRId64" audio stream time base units\n", new_audio_ts); + pout("unable to seek to %"PRId64" audio filter time base units/%"PRId64" audio stream time base units\n", new_audio_filt_ts, new_audio_st_ts); goto try_restore_audio; } - pout("global seek using audio seek to %"PRId64" audio stream time base units\n", new_audio_ts); + pout("global seek using audio seek to %"PRId64" audio filter time base units/%"PRId64" audio stream time base units\n", new_audio_filt_ts, new_audio_st_ts); flush: npv_video_dec_flush(); npv_audio_dec_flush(); @@ -682,9 +680,12 @@ flush: return; try_restore_audio: - a = avformat_seek_pkt(npv_fmt_ctx_p, npv_audio_st_p.id, audio_now, 0); + now_audio_st_ts = avutil_rescale_q(now_audio_filt_ts, audio_filt_tb, + npv_audio_st_p.tb); + a = avformat_seek_pkt(npv_fmt_ctx_p, npv_audio_st_p.id, now_audio_st_ts, + 0); if (a < 0) /* we don't send an application error */ - exit_ok("unable to restore audio to %"PRId64" audio stream time base units\n", audio_now); + exit_ok("unable to restore audio to %"PRId64" audio filter time base units/%"PRId64" audio stream time base units\n", now_audio_filt_ts, now_audio_st_ts); goto flush; } #undef TS_FROM_CLK_OK diff --git a/npv/main.c b/npv/main.c index 1bb6e6e..ffef877 100644 --- a/npv/main.c +++ b/npv/main.c @@ -27,6 +27,7 @@ #include #include #include +#include #include /*---------------------------------------------------------------------------*/ #include "npv/c_fixing.h" diff --git a/npv/namespace/ffmpeg.h b/npv/namespace/ffmpeg.h index 44451b6..1e1b61b 100644 --- a/npv/namespace/ffmpeg.h +++ b/npv/namespace/ffmpeg.h @@ -35,6 +35,7 @@ #define avcodec_pkt_ref_alloc av_packet_alloc #define avcodec_pkt_ref_t AVPacket #define avcodec_pkt_unref av_packet_unref +#define avfilter_bufsink_tb_get av_buffersink_get_time_base #define avfilter_filt_t AVFilter #define avfilter_filt_ctx_t AVFilterContext #define avfilter_filt_graph_t AVFilterGraph @@ -114,6 +115,7 @@ #undef avcodec_pkt_ref_alloc #undef avcodec_pkt_ref_t #undef avcodec_pkt_unref +#undef avfilter_bufsink_tb_get #undef avfilter_filt_t #undef avfilter_filt_ctx_t #undef avfilter_filt_graph_t diff --git a/npv/pipeline/local/code.frag.c b/npv/pipeline/local/code.frag.c index 0fa7a21..52d64ed 100644 --- a/npv/pipeline/local/code.frag.c +++ b/npv/pipeline/local/code.frag.c @@ -80,16 +80,15 @@ STATIC void *read_thd_entry(void *arg) /* * our alsa audio buf is (rate/4)~0.25s (interactivity like vol processing). * then we would like to have ~2 times this buf (~double buf) of decoded audio - * frs. namely we try to have ~(2 *0.25s) of decoded audio frames. we presume - * the audio dec is fine grained enough in order to base our calculation on the - * pts-es of 2 sets of audio frs. + * frs. namely we try to have ~(2 *0.25s) of decoded audio frs. + * due to discontinuity with ts and change of audio properties, we account for + * the total amount of us based on the rate and frs n. */ STATIC bool have_enough_predecoded_audio_frs(void) { - int64_t pts_min; - int64_t pts_max; - int64_t pts_delta; - int64_t pts_delta_limit; + u32 set; + int64_t total_us; + int64_t total_us_threshold; npv_audio_dec_sets_lock(); if (npv_audio_dec_sets_p.eof_receive) { @@ -100,23 +99,27 @@ STATIC bool have_enough_predecoded_audio_frs(void) return false; } /* from here we have at least 1 sets of audio frs */ - /* we presume the audio sets are in pts order */ - pts_min = npv_audio_dec_sets_p.a[0]->pts; - pts_max = npv_audio_dec_sets_p.a[npv_audio_dec_sets_p.n - 1]->pts; + set = 0; + total_us = 0; + loop { + if (set == (npv_audio_dec_sets_p.n - 1)) + break; + total_us += npv_audio_dec_sets_p.a[set]->frs_n * 1000000 + / npv_audio_dec_sets_p.a[set]->fr_rate; + ++set; + } npv_audio_dec_sets_unlock(); - - pts_delta = pts_max - pts_min; - /* 2 * 0.25s = 500 ms */ - pts_delta_limit = 500 * npv_audio_st_p.tb.den / npv_audio_st_p.tb.num - / 1000; - if (pts_delta >= pts_delta_limit) + /* 250 * 2 ms */ + total_us_threshold = 250 * 2 * 1000; + if (total_us >= total_us_threshold) return true; return false; } STATIC void audio(void) { loop /* infinite loop */ { if (have_enough_predecoded_audio_frs()) { - wait(250000000); /* (rate/4)~0.25s */ + /* human ear is sensitive to ~20ms crack, make it 16ms */ + wait(16000000); continue; } /* can be long, finer-grained locking is done in there */ @@ -139,46 +142,37 @@ STATIC void *audio_thd_entry(void *arg) /* unreachable */ } /* - * same heuristics than for audio. there is some sort of sync due to video fr - * dropping based on "audio now" in the main thd + * predecoded video frs are expensive from the perspectives of the cpu and the + * mem hierarchy. + * 1920 * 1080 * 4 bytes = 8MB. I like powers of 2, then 4. */ STATIC bool have_enough_predecoded_video_frs(void) { - int64_t pts_min; - int64_t pts_max; - int64_t pts_delta; - int64_t pts_delta_limit; + bool r; npv_video_dec_frs_lock(); if (npv_video_dec_frs_p.eof_receive) { - npv_video_dec_frs_unlock(); - return true; - } else if (npv_video_dec_frs_p.n == 0) { - npv_video_dec_frs_unlock(); - return false; - } - /* from here we have at least 1 video fr */ - /* we presume the video frs are in pts order */ - pts_min = npv_video_dec_frs_p.a[0]->pts; - pts_max = npv_video_dec_frs_p.a[npv_video_dec_frs_p.n - 1]->pts; + r = true; + } else if (npv_video_dec_frs_p.n < 4) { + r = false; + } else /* >= 4 */ + r = true; npv_video_dec_frs_unlock(); - - pts_delta = pts_max - pts_min; - /* 2 * 0.25s = 500 ms */ - pts_delta_limit = 500 * npv_video_st_p.tb.den / npv_video_st_p.tb.num - / 1000; - if (pts_delta >= pts_delta_limit) - return true; - return false; + return r; } STATIC void video(void) { loop /* infinite loop */ { if (have_enough_predecoded_video_frs()) { - wait(250000000); /* (audio rate/4)~0.25s */ + wait(8000000); /* 60fps or 16ms, let's check every 8ms */ continue; } - /* can be long, finer-grained locking is done in there */ - npv_video_pkts_send(); + /* + * can be long, finer-grained locking is done in there. filling the dec + * bufs should be "faster" than decoding, and we may not have enough + * pkts to fill those bufs, then it should exit without filling up + * mem. + */ + npv_video_pkts_send(); /* fill the dec bufs */ npv_video_dec_frs_receive_avail(); }} STATIC void *video_thd_entry(void *arg) diff --git a/npv/pipeline/namespace/main.c b/npv/pipeline/namespace/main.c index 35f28c2..df1bddf 100644 --- a/npv/pipeline/namespace/main.c +++ b/npv/pipeline/namespace/main.c @@ -1,7 +1,9 @@ #ifndef CLEANUP /*----------------------------------------------------------------------------*/ /* some struct fields */ -#define sz size +#define frs_n nb_samples +#define fr_rate sample_rate +#define sz size /*----------------------------------------------------------------------------*/ #define audio npv_pipeline_audio #define audio_thd_entry npv_pipeline_audio_thd_entry @@ -34,6 +36,8 @@ #undef video_thd_entry #undef wait /*----------------------------------------------------------------------------*/ +#undef frs_n +#undef fr_rate #undef sz /*----------------------------------------------------------------------------*/ #endif diff --git a/npv/video/local/code.frag.c b/npv/video/local/code.frag.c index 48ef6ab..71655cc 100644 --- a/npv/video/local/code.frag.c +++ b/npv/video/local/code.frag.c @@ -335,8 +335,12 @@ STATIC void frs_drop(s64 now) s64 threshold; u16 fr; - /* audio can be late up to 0.25s, and audio is 'now' */ - threshold = (250 * st_p.tb.den) / (st_p.tb.num * 1000); + /* + * https://en.wikipedia.org/wiki/Audio_to_video_synchronization + * we need to make room in the dec a of predecoded frs, based on the + * previous url (20200922), we abitrary choose -30ms. + */ + threshold = (30 * st_p.tb.den) / (st_p.tb.num * 1000); low = now - threshold; fr = 0; loop { @@ -348,7 +352,6 @@ STATIC void frs_drop(s64 now) pts = dec_frs_p.a[fr]->pts; fr_priv = dec_frs_p.priv_a[fr]; - /* keep the fr the scaler is related to */ if ((dec_frs_p.a[fr] != scaler_p.img.fr) && (pts < low)) { if (dec_frs_p.a[fr] == last_fr_sent_to_pe_l) -- 2.11.4.GIT