From 05acb78a01ca9262744f34981b5ef54fd7b3ba28 Mon Sep 17 00:00:00 2001 From: Benjamin Otte Date: Fri, 26 Oct 2007 21:13:20 +0200 Subject: [PATCH] rewrite GStreamer backend to directly use the element and not create a pipeline That was Wim's suggested method, and it works fine. The only issue right now is that GStreamer also does the resampling, and a) audioresample is slow and b) the whole process takes a lot of time. This causes the app to stop for half a second when a long sample gets decoded. Not good. --- libswfdec/swfdec_codec_gst.c | 525 +++++++++++++++++++++++++++---------------- 1 file changed, 326 insertions(+), 199 deletions(-) diff --git a/libswfdec/swfdec_codec_gst.c b/libswfdec/swfdec_codec_gst.c index 08cf154a..bd8f95a9 100644 --- a/libswfdec/swfdec_codec_gst.c +++ b/libswfdec/swfdec_codec_gst.c @@ -28,273 +28,400 @@ #include "swfdec_debug.h" #include "swfdec_internal.h" -#if 0 -#define swfdec_cond_wait(cond, mutex) G_STMT_START { \ - g_print ("waiting at %s\n", G_STRLOC); \ - g_cond_wait (cond, mutex); \ - g_print (" done at %s\n", G_STRLOC); \ -}G_STMT_END -#else -#define swfdec_cond_wait g_cond_wait -#endif +/*** BUFFER ***/ -/*** AUDIO ***/ +static void +swfdec_gst_buffer_free (unsigned char *data, gpointer priv) +{ + gst_buffer_unref (priv); +} -typedef struct _SwfdecGstAudio SwfdecGstAudio; -struct _SwfdecGstAudio { - SwfdecAudioDecoder decoder; +static SwfdecBuffer * +swfdec_buffer_new_from_gst (GstBuffer *buffer) +{ + SwfdecBuffer *ret; - GMutex * mutex; /* mutex that blocks everything below */ - GCond * cond; /* cond used to signal when stuff below changes */ - volatile int refcount; /* refcount (d'oh) */ + g_return_val_if_fail (GST_IS_BUFFER (buffer), NULL); - GstElement * pipeline; /* pipeline that is playing or NULL when done */ - SwfdecBuffer * in; /* next input buffer or NULL */ - SwfdecBufferQueue * out; /* all the stored output buffers */ - GstCaps * srccaps; /* caps to set on buffers */ - gboolean eof; /* we've pushed EOF */ - gboolean done; /* TRUE after decoding stopped (error or EOF) */ -}; + ret = swfdec_buffer_new (); + ret->data = GST_BUFFER_DATA (buffer); + ret->length = GST_BUFFER_SIZE (buffer); + ret->free = swfdec_gst_buffer_free; + ret->priv = buffer; -static void -swfdec_gst_audio_unref (gpointer data, GObject *unused) + return ret; +} + +static GstBuffer * +swfdec_gst_buffer_new (SwfdecBuffer *buffer) { - SwfdecGstAudio *player = data; + /* FIXME: make this a zero-copy operation */ + GstBuffer *ret; - if (!g_atomic_int_dec_and_test (&player->refcount)) - return; - g_cond_free (player->cond); - g_mutex_free (player->mutex); - gst_caps_unref (player->srccaps); - if (player->in) - swfdec_buffer_unref (player->in); - swfdec_buffer_queue_unref (player->out); - g_slice_free (SwfdecGstAudio, player); + g_return_val_if_fail (buffer != NULL , NULL); + + ret = gst_buffer_new_and_alloc (buffer->length); + memcpy (GST_BUFFER_DATA (ret), buffer->data, buffer->length); + + return ret; } -static void -swfdec_audio_decoder_gst_free (SwfdecAudioDecoder *dec) +/*** TYPEFINDING ***/ + +/* NB: try to mirror decodebin behavior */ +static gboolean +swfdec_gst_feature_filter (GstPluginFeature *feature, gpointer caps) { - SwfdecGstAudio *player = (SwfdecGstAudio *) dec; - GstElement *pipeline; + const GList *walk; + const gchar *klass; - g_mutex_lock (player->mutex); - pipeline = player->pipeline; - player->pipeline = NULL; - g_cond_signal (player->cond); - g_mutex_unlock (player->mutex); - gst_element_set_state (pipeline, GST_STATE_NULL); - g_object_unref (pipeline); + /* we only care about element factories */ + if (!GST_IS_ELEMENT_FACTORY (feature)) + return FALSE; + + /* only decoders are interesting */ + klass = gst_element_factory_get_klass (GST_ELEMENT_FACTORY (feature)); + if (strstr (klass, "Decoder") == NULL) + return FALSE; + + /* only select elements with autoplugging rank */ + if (gst_plugin_feature_get_rank (feature) < GST_RANK_MARGINAL) + return FALSE; + + /* only care about the right sink caps */ + for (walk = gst_element_factory_get_static_pad_templates (GST_ELEMENT_FACTORY (feature)); + walk; walk = walk->next) { + GstStaticPadTemplate *template = walk->data; + GstCaps *intersect; + GstCaps *template_caps; + + if (template->direction != GST_PAD_SINK) + continue; + + template_caps = gst_static_caps_get (&template->static_caps); + intersect = gst_caps_intersect (caps, template_caps); + + gst_caps_unref (template_caps); + if (gst_caps_is_empty (intersect)) { + gst_caps_unref (intersect); + } else { + gst_caps_unref (intersect); + return TRUE; + } + } + return FALSE; +} + +static int +swfdec_gst_compare_features (gconstpointer a_, gconstpointer b_) +{ + int diff; + GstPluginFeature *a = GST_PLUGIN_FEATURE (a_); + GstPluginFeature *b = GST_PLUGIN_FEATURE (b_); - swfdec_gst_audio_unref (player, NULL); + diff = gst_plugin_feature_get_rank (b) - gst_plugin_feature_get_rank (a); + if (diff != 0) + return diff; + + return strcmp (gst_plugin_feature_get_name (a), gst_plugin_feature_get_name (b)); } -static void -swfdec_audio_decoder_gst_push (SwfdecAudioDecoder *dec, SwfdecBuffer *buffer) +static GstElement * +swfdec_gst_get_element (GstCaps *caps) { - SwfdecGstAudio *player = (SwfdecGstAudio *) dec; + GstElement *element; + GList *list; - g_mutex_lock (player->mutex); - g_return_if_fail (!player->eof); - while (player->in != NULL && !player->done) { - swfdec_cond_wait (player->cond, player->mutex); - } - if (buffer) { - player->in = swfdec_buffer_ref (buffer); - } else { - player->eof = TRUE; - } - g_cond_signal (player->cond); - g_mutex_unlock (player->mutex); + list = gst_registry_feature_filter (gst_registry_get_default (), + swfdec_gst_feature_filter, FALSE, caps); + if (list == NULL) + return NULL; + + list = g_list_sort (list, swfdec_gst_compare_features); + element = gst_element_factory_create (list->data, "decoder"); + gst_plugin_feature_list_free (list); + return element; } -static SwfdecBuffer * -swfdec_audio_decoder_gst_pull (SwfdecAudioDecoder *dec) +/*** PADS ***/ + +static GstPad * +swfdec_gst_connect_srcpad (GstElement *element, GstCaps *caps) { - SwfdecGstAudio *player = (SwfdecGstAudio *) dec; - SwfdecBuffer *buffer; + GstPad *srcpad, *sinkpad; - g_mutex_lock (player->mutex); - if (player->eof) { - while (!player->done) - swfdec_cond_wait (player->cond, player->mutex); - } - buffer = swfdec_buffer_queue_pull_buffer (player->out); - g_mutex_unlock (player->mutex); - return buffer; + sinkpad = gst_element_get_pad (element, "sink"); + if (sinkpad == NULL) + return NULL; + srcpad = gst_pad_new ("src", GST_PAD_SRC); + if (!gst_pad_set_caps (srcpad, caps)) + goto error; + if (gst_pad_link (srcpad, sinkpad) != GST_PAD_LINK_OK) + goto error; + + gst_object_unref (sinkpad); + gst_pad_set_active (srcpad, TRUE); + return srcpad; + +error: + SWFDEC_ERROR ("failed to create or link srcpad"); + gst_object_unref (sinkpad); + gst_object_unref (srcpad); + return NULL; } -static void -swfdec_audio_decoder_gst_fakesrc_handoff (GstElement *fakesrc, GstBuffer *buf, - GstPad *pad, SwfdecGstAudio *player) +static GstPad * +swfdec_gst_connect_sinkpad (GstElement *element, GstCaps *caps) { - g_mutex_lock (player->mutex); - while (player->pipeline != NULL && player->in == NULL && player->eof == FALSE) - swfdec_cond_wait (player->cond, player->mutex); - if (player->pipeline == NULL) { - g_mutex_unlock (player->mutex); - return; + GstPad *srcpad, *sinkpad; + + srcpad = gst_element_get_pad (element, "src"); + if (srcpad == NULL) + return NULL; + sinkpad = gst_pad_new ("sink", GST_PAD_SINK); + if (!gst_pad_set_caps (sinkpad, caps)) + goto error; + if (gst_pad_link (srcpad, sinkpad) != GST_PAD_LINK_OK) + goto error; + + gst_object_unref (srcpad); + gst_pad_set_active (sinkpad, TRUE); + return sinkpad; + +error: + SWFDEC_ERROR ("failed to create or link sinkpad"); + gst_object_unref (srcpad); + gst_object_unref (sinkpad); + return NULL; +} + +/*** DECODER ***/ + +typedef struct { + GstElement * decoder; + GstPad * src; + GstPad * sink; + GQueue * queue; /* all the stored output GstBuffers */ +} SwfdecGstDecoder; + +static GstFlowReturn +swfdec_gst_chain_func (GstPad *pad, GstBuffer *buffer) +{ + GQueue *queue = g_object_get_data (G_OBJECT (pad), "swfdec-queue"); + + g_queue_push_tail (queue, buffer); + + return GST_FLOW_OK; +} + +static gboolean +swfdec_gst_decoder_init (SwfdecGstDecoder *dec, const char *name, GstCaps *srccaps, GstCaps *sinkcaps) +{ + if (name) { + dec->decoder = gst_element_factory_make (name, "decoder"); + } else { + dec->decoder = swfdec_gst_get_element (srccaps); } - if (player->eof) { - //doesn't work: g_object_set (fakesrc, "num-buffers", 1, NULL); - /* HACK: just tell everyone we're done, that'll probably lose data in the - * gst stream, since we can't properly push EOF, but that's life... */ - player->done = TRUE; + if (dec->decoder == NULL) { + SWFDEC_ERROR ("failed to create decoder"); + return FALSE; } - if (player->in) { - buf->data = g_memdup (player->in->data, player->in->length); - buf->malloc_data = buf->data; - buf->size = player->in->length; + dec->src = swfdec_gst_connect_srcpad (dec->decoder, srccaps); + if (dec->src == NULL) + return FALSE; + dec->sink = swfdec_gst_connect_sinkpad (dec->decoder, sinkcaps); + if (dec->sink == NULL) + return FALSE; + gst_pad_set_chain_function (dec->sink, swfdec_gst_chain_func); + dec->queue = g_queue_new (); + g_object_set_data (G_OBJECT (dec->sink), "swfdec-queue", dec->queue); + if (!gst_element_set_state (dec->decoder, GST_STATE_PLAYING) == GST_STATE_CHANGE_SUCCESS) { + SWFDEC_ERROR ("could not change element state"); + return FALSE; } - gst_buffer_set_caps (buf, player->srccaps); - swfdec_buffer_unref (player->in); - player->in = NULL; - g_cond_signal (player->cond); - g_mutex_unlock (player->mutex); + return TRUE; } static void -swfdec_audio_decoder_gst_fakesink_handoff (GstElement *fakesrc, GstBuffer *buf, - GstPad *pad, SwfdecGstAudio *player) +swfdec_gst_decoder_finish (SwfdecGstDecoder *dec) { - SwfdecBuffer *buffer; + if (dec->decoder) { + gst_element_set_state (dec->decoder, GST_STATE_NULL); + g_object_unref (dec->decoder); + dec->decoder = NULL; + } + if (dec->src) { + g_object_unref (dec->src); + dec->src = NULL; + } + if (dec->sink) { + g_object_unref (dec->sink); + dec->sink = NULL; + } + if (dec->queue) { + GstBuffer *buffer; + while ((buffer = g_queue_pop_head (dec->queue)) != NULL) { + gst_buffer_unref (buffer); + } + g_queue_free (dec->queue); + dec->queue = NULL; + } +} - g_mutex_lock (player->mutex); +static gboolean +swfdec_gst_decoder_push (SwfdecGstDecoder *dec, GstBuffer *buffer) +{ + return GST_FLOW_IS_SUCCESS (gst_pad_push (dec->src, buffer)); +} - while (player->pipeline == NULL && player->out != NULL) - swfdec_cond_wait (player->cond, player->mutex); - buffer = swfdec_buffer_new_for_data ( - g_memdup (buf->data, buf->size), buf->size); - swfdec_buffer_queue_push (player->out, buffer); - g_cond_signal (player->cond); - g_mutex_unlock (player->mutex); +static void +swfdec_gst_decoder_push_eos (SwfdecGstDecoder *dec) +{ + gst_pad_push_event (dec->src, gst_event_new_eos ()); } +static GstBuffer * +swfdec_gst_decoder_pull (SwfdecGstDecoder *dec) +{ + return g_queue_pop_head (dec->queue); +} + +/*** AUDIO ***/ + +typedef struct _SwfdecGstAudio SwfdecGstAudio; +struct _SwfdecGstAudio { + SwfdecAudioDecoder decoder; + + gboolean error; + SwfdecGstDecoder dec; + SwfdecGstDecoder convert; + SwfdecGstDecoder resample; +}; + static void -swfdec_audio_decoder_gst_link (GstElement *src, GstPad *pad, GstElement *sink) +swfdec_audio_decoder_gst_free (SwfdecAudioDecoder *dec) { - if (!gst_element_link (src, sink)) { - SWFDEC_ERROR ("no delayed link"); - } + SwfdecGstAudio *player = (SwfdecGstAudio *) dec; + + swfdec_gst_decoder_finish (&player->dec); + swfdec_gst_decoder_finish (&player->convert); + swfdec_gst_decoder_finish (&player->resample); + + g_slice_free (SwfdecGstAudio, player); } -static GstBusSyncReply -swfdec_audio_decoder_gst_handle_bus (GstBus *bus, GstMessage *message, gpointer data) +static void +swfdec_audio_decoder_gst_push (SwfdecAudioDecoder *dec, SwfdecBuffer *buffer) { - SwfdecGstAudio *player = data; - - switch (message->type) { - case GST_MESSAGE_EOS: - case GST_MESSAGE_ERROR: - g_mutex_lock (player->mutex); - g_cond_signal (player->cond); - player->done = TRUE; - g_mutex_unlock (player->mutex); - break; - default: - break; + SwfdecGstAudio *player = (SwfdecGstAudio *) dec; + GstBuffer *buf; + + if (player->error) + return; + if (buffer == NULL) { + swfdec_gst_decoder_push_eos (&player->dec); + } else { + swfdec_buffer_ref (buffer); + buf = swfdec_gst_buffer_new (buffer); + if (!swfdec_gst_decoder_push (&player->dec, buf)) + goto error; } - return GST_BUS_PASS; + while ((buf = swfdec_gst_decoder_pull (&player->dec))) { + if (!swfdec_gst_decoder_push (&player->convert, buf)) + goto error; + } + while ((buf = swfdec_gst_decoder_pull (&player->convert))) { + if (!swfdec_gst_decoder_push (&player->resample, buf)) + goto error; + } + return; + +error: + SWFDEC_ERROR ("error pushing"); + player->error = TRUE; +} + +static SwfdecBuffer * +swfdec_audio_decoder_gst_pull (SwfdecAudioDecoder *dec) +{ + SwfdecGstAudio *player = (SwfdecGstAudio *) dec; + GstBuffer *buf; + + if (player->error) + return NULL; + buf = swfdec_gst_decoder_pull (&player->resample); + if (buf == NULL) + return NULL; + return swfdec_buffer_new_from_gst (buf); } SwfdecAudioDecoder * swfdec_audio_decoder_gst_new (SwfdecAudioCodec type, SwfdecAudioFormat format) { SwfdecGstAudio *player; - GstElement *fakesrc, *fakesink, *decoder, *convert; - GstBus *bus; - GstCaps *caps; + GstCaps *srccaps, *sinkcaps; if (!gst_init_check (NULL, NULL, NULL)) return NULL; switch (type) { case SWFDEC_AUDIO_CODEC_MP3: - caps = gst_caps_from_string ("audio/mpeg, mpegversion=(int)1, layer=(int)3"); + srccaps = gst_caps_from_string ("audio/mpeg, mpegversion=(int)1, layer=(int)3"); break; default: return NULL; } - g_assert (caps); + g_assert (srccaps); player = g_slice_new0 (SwfdecGstAudio); player->decoder.format = swfdec_audio_format_new (44100, 2, TRUE); player->decoder.pull = swfdec_audio_decoder_gst_pull; player->decoder.push = swfdec_audio_decoder_gst_push; player->decoder.free = swfdec_audio_decoder_gst_free; - player->pipeline = gst_pipeline_new ("pipeline"); - player->refcount = 1; - g_assert (player->pipeline); - bus = gst_element_get_bus (player->pipeline); - g_atomic_int_inc (&player->refcount); - g_object_weak_ref (G_OBJECT (bus), swfdec_gst_audio_unref, player); - gst_bus_set_sync_handler (bus, swfdec_audio_decoder_gst_handle_bus, player); - player->mutex = g_mutex_new (); - player->cond = g_cond_new (); - player->out = swfdec_buffer_queue_new (); - player->srccaps = caps; - fakesrc = gst_element_factory_make ("fakesrc", NULL); - if (fakesrc == NULL) { - SWFDEC_ERROR ("failed to create fakesrc"); - swfdec_audio_decoder_gst_free (&player->decoder); - return NULL; - } - g_object_set (fakesrc, "signal-handoffs", TRUE, - "can-activate-pull", FALSE, NULL); - g_signal_connect (fakesrc, "handoff", - G_CALLBACK (swfdec_audio_decoder_gst_fakesrc_handoff), player); - g_atomic_int_inc (&player->refcount); - g_object_weak_ref (G_OBJECT (fakesrc), swfdec_gst_audio_unref, player); - gst_bin_add (GST_BIN (player->pipeline), fakesrc); - fakesink = gst_element_factory_make ("fakesink", NULL); - if (fakesink == NULL) { - SWFDEC_ERROR ("failed to create fakesink"); - swfdec_audio_decoder_gst_free (&player->decoder); - return NULL; - } - g_object_set (fakesink, "signal-handoffs", TRUE, NULL); - g_signal_connect (fakesink, "handoff", - G_CALLBACK (swfdec_audio_decoder_gst_fakesink_handoff), player); - g_atomic_int_inc (&player->refcount); - g_object_weak_ref (G_OBJECT (fakesink), swfdec_gst_audio_unref, player); - gst_bin_add (GST_BIN (player->pipeline), fakesink); - decoder = gst_element_factory_make ("decodebin", NULL); - if (decoder == NULL) { - SWFDEC_ERROR ("failed to create decoder"); - swfdec_audio_decoder_gst_free (&player->decoder); - return NULL; - } - gst_bin_add (GST_BIN (player->pipeline), decoder); - convert = gst_element_factory_make ("audioconvert", NULL); - if (convert == NULL) { - SWFDEC_ERROR ("failed to create audioconvert"); - swfdec_audio_decoder_gst_free (&player->decoder); - return NULL; - } - gst_bin_add (GST_BIN (player->pipeline), convert); - g_signal_connect (decoder, "pad-added", - G_CALLBACK (swfdec_audio_decoder_gst_link), convert); - - caps = gst_caps_from_string ("audio/x-raw-int, endianness=byte_order, signed=(boolean)true, width=16, depth=16, rate=44100, channels=2"); - g_assert (caps); - if (!gst_element_link_filtered (fakesrc, decoder, player->srccaps) || - !gst_element_link_filtered (convert, fakesink, caps)) { - SWFDEC_ERROR ("linking failed"); - swfdec_audio_decoder_gst_free (&player->decoder); - return NULL; - } - gst_caps_unref (caps); - if (gst_element_set_state (player->pipeline, GST_STATE_PLAYING) == GST_STATE_CHANGE_FAILURE) { - SWFDEC_ERROR ("failed to change sate"); - swfdec_audio_decoder_gst_free (&player->decoder); - return NULL; - } + /* create decoder */ + sinkcaps = gst_caps_from_string ("audio/x-raw-int"); + g_assert (sinkcaps); + if (!swfdec_gst_decoder_init (&player->dec, NULL, srccaps, sinkcaps)) + goto error; + /* create audioconvert */ + gst_caps_unref (srccaps); + srccaps = sinkcaps; + sinkcaps = gst_caps_from_string ("audio/x-raw-int, endianness=byte_order, signed=(boolean)true, width=16, depth=16, channels=2"); + g_assert (sinkcaps); + if (!swfdec_gst_decoder_init (&player->convert, "audioconvert", srccaps, sinkcaps)) + goto error; + /* create audiorate */ + gst_caps_unref (srccaps); + srccaps = sinkcaps; + sinkcaps = gst_caps_from_string ("audio/x-raw-int, endianness=byte_order, signed=(boolean)true, width=16, depth=16, rate=44100, channels=2"); + g_assert (sinkcaps); + if (!swfdec_gst_decoder_init (&player->resample, "audioresample", srccaps, sinkcaps)) + goto error; + + gst_caps_unref (srccaps); + gst_caps_unref (sinkcaps); return &player->decoder; + +error: + swfdec_audio_decoder_gst_free (&player->decoder); + gst_caps_unref (srccaps); + gst_caps_unref (sinkcaps); + return NULL; } /*** VIDEO ***/ +#if 0 +#define swfdec_cond_wait(cond, mutex) G_STMT_START { \ + g_print ("waiting at %s\n", G_STRLOC); \ + g_cond_wait (cond, mutex); \ + g_print (" done at %s\n", G_STRLOC); \ +}G_STMT_END +#else +#define swfdec_cond_wait g_cond_wait +#endif + typedef struct _SwfdecGstVideo SwfdecGstVideo; struct _SwfdecGstVideo { SwfdecVideoDecoder decoder; -- 2.11.4.GIT