From 7fd02feb6c6be674066b68c2f53776a5504f4657 Mon Sep 17 00:00:00 2001
From: Thomas Guillem <thomas@gllm.fr>
Date: Mon, 19 Mar 2018 17:39:05 +0100
Subject: [PATCH] chroma: chain: handle resize converter fmt changes

This fixes the transform filter from the chain converter. Consequently, this
fixes transform filters with HW decoders.

Recent improvements in copy_cache_t allow to have correct performances for most
use cases. This is far from perfect, but doing 4K HW decoding and SW transform
is slightly better than doing 4K SW decoding and SW transform (tested on
Linux/Windows).

Obviously, the best solution is to do this transformation with shaders. This
will be possible in VLC 4.0 with future gl/d3d filters API.

Fixes #19096
---
 modules/video_chroma/chain.c | 53 ++++++++++++++++++++++++++++++++++++--------
 1 file changed, 44 insertions(+), 9 deletions(-)

diff --git a/modules/video_chroma/chain.c b/modules/video_chroma/chain.c
index 3542559679..bfd8f50e8a 100644
--- a/modules/video_chroma/chain.c
+++ b/modules/video_chroma/chain.c
@@ -62,6 +62,7 @@ static int BuildChromaChain( filter_t *p_filter );
 static int BuildFilterChain( filter_t *p_filter );
 
 static int CreateChain( filter_t *p_parent, const es_format_t *p_fmt_mid );
+static int CreateResizeChromaChain( filter_t *p_parent, const es_format_t *p_fmt_mid );
 static filter_t * AppendTransform( filter_chain_t *p_chain, const es_format_t *p_fmt_in,
                                    const es_format_t *p_fmt_out );
 static void EsFormatMergeSize( es_format_t *p_dst,
@@ -160,7 +161,7 @@ static int Activate( filter_t *p_filter, int (*pf_build)(filter_t *) )
         },
     };
 
-    p_sys->p_chain = filter_chain_NewVideo( p_filter, false, &owner );
+    p_sys->p_chain = filter_chain_NewVideo( p_filter, p_filter->b_allow_fmt_out_change, &owner );
     if( !p_sys->p_chain )
     {
         free( p_sys );
@@ -193,6 +194,12 @@ static int Activate( filter_t *p_filter, int (*pf_build)(filter_t *) )
         free( p_sys );
         return VLC_EGENERIC;
     }
+    else if( p_filter->b_allow_fmt_out_change )
+    {
+        es_format_Clean( &p_filter->fmt_out );
+        es_format_Copy( &p_filter->fmt_out,
+                        filter_chain_GetFmtOut( p_filter->p_sys->p_chain ) );
+    }
     /* */
     p_filter->pf_video_filter = Chain;
     return VLC_SUCCESS;
@@ -291,8 +298,9 @@ static int BuildChromaResize( filter_t *p_filter )
     /* Lets try resizing and then doing the chroma conversion */
     msg_Dbg( p_filter, "Trying to build resize+chroma" );
     EsFormatMergeSize( &fmt_mid, &p_filter->fmt_in, &p_filter->fmt_out );
-    i_ret = CreateChain( p_filter, &fmt_mid );
+    i_ret = CreateResizeChromaChain( p_filter, &fmt_mid );
     es_format_Clean( &fmt_mid );
+
     if( i_ret == VLC_SUCCESS )
         return VLC_SUCCESS;
 
@@ -397,13 +405,7 @@ static int BuildFilterChain( filter_t *p_filter )
         }
         es_format_Clean( &fmt_mid );
     }
-    if (i_ret == VLC_SUCCESS)
-    {
-        es_format_Clean( &p_filter->fmt_out );
-        es_format_Copy( &p_filter->fmt_out,
-                        filter_chain_GetFmtOut( p_filter->p_sys->p_chain ) );
-    }
-    else
+    if( i_ret != VLC_SUCCESS )
         filter_chain_Reset( p_filter->p_sys->p_chain, &p_filter->fmt_in, &p_filter->fmt_out );
 
     return i_ret;
@@ -453,6 +455,39 @@ error:
     return VLC_EGENERIC;
 }
 
+static int CreateResizeChromaChain( filter_t *p_parent, const es_format_t *p_fmt_mid )
+{
+    filter_chain_Reset( p_parent->p_sys->p_chain, &p_parent->fmt_in, &p_parent->fmt_out );
+
+    int i_ret = filter_chain_AppendConverter( p_parent->p_sys->p_chain,
+                                              NULL, p_fmt_mid );
+    if( i_ret != VLC_SUCCESS )
+        return i_ret;
+
+    if( p_parent->b_allow_fmt_out_change )
+    {
+        /* XXX: Update i_sar_num/i_sar_den from last converter. Cf.
+         * p_filter->b_allow_fmt_out_change comment in video_chroma/swscale.c.
+         * */
+
+        es_format_t fmt_out;
+        es_format_Copy( &fmt_out,
+                        filter_chain_GetFmtOut( p_parent->p_sys->p_chain ) );
+        fmt_out.video.i_chroma = p_parent->fmt_out.video.i_chroma;
+
+        i_ret = filter_chain_AppendConverter( p_parent->p_sys->p_chain,
+                                              NULL, &fmt_out );
+        es_format_Clean( &fmt_out );
+    }
+    else
+        i_ret = filter_chain_AppendConverter( p_parent->p_sys->p_chain,
+                                              NULL, &p_parent->fmt_out );
+
+    if( i_ret != VLC_SUCCESS )
+        filter_chain_Reset( p_parent->p_sys->p_chain, NULL, NULL );
+    return i_ret;
+}
+
 static filter_t * AppendTransform( filter_chain_t *p_chain, const es_format_t *p_fmt1,
                                    const es_format_t *p_fmt2 )
 {
-- 
2.11.4.GIT