2 * GStreamer transform backend
4 * Copyright 2022 RĂ©mi Bernon for CodeWeavers
6 * This library is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU Lesser General Public
8 * License as published by the Free Software Foundation; either
9 * version 2.1 of the License, or (at your option) any later version.
11 * This library is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 * Lesser General Public License for more details.
16 * You should have received a copy of the GNU Lesser General Public
17 * License along with this library; if not, write to the Free Software
18 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
32 #include <gst/video/video.h>
33 #include <gst/audio/audio.h>
36 #define WIN32_NO_STATUS
40 #include "unix_private.h"
42 GST_DEBUG_CATEGORY_EXTERN(wine
);
43 #define GST_CAT_DEFAULT wine
47 GstElement
*container
;
48 GstPad
*my_src
, *my_sink
;
49 GstPad
*their_sink
, *their_src
;
52 guint input_max_length
;
53 GstAtomicQueue
*output_queue
;
54 GstBuffer
*output_buffer
;
57 static GstFlowReturn
transform_sink_chain_cb(GstPad
*pad
, GstObject
*parent
, GstBuffer
*buffer
)
59 struct wg_transform
*transform
= gst_pad_get_element_private(pad
);
61 GST_LOG("transform %p, buffer %p.", transform
, buffer
);
63 gst_atomic_queue_push(transform
->output_queue
, buffer
);
68 NTSTATUS
wg_transform_destroy(void *args
)
70 struct wg_transform
*transform
= args
;
74 gst_buffer_list_unref(transform
->input
);
76 gst_element_set_state(transform
->container
, GST_STATE_NULL
);
78 if (transform
->output_buffer
)
79 gst_buffer_unref(transform
->output_buffer
);
80 while ((buffer
= gst_atomic_queue_pop(transform
->output_queue
)))
81 gst_buffer_unref(buffer
);
83 g_object_unref(transform
->their_sink
);
84 g_object_unref(transform
->their_src
);
85 g_object_unref(transform
->container
);
86 g_object_unref(transform
->my_sink
);
87 g_object_unref(transform
->my_src
);
88 gst_atomic_queue_unref(transform
->output_queue
);
91 return STATUS_SUCCESS
;
94 static GstElement
*transform_find_element(GstElementFactoryListType type
, GstCaps
*src_caps
, GstCaps
*sink_caps
)
96 GstElement
*element
= NULL
;
97 GList
*tmp
, *transforms
;
100 if (!(transforms
= gst_element_factory_list_get_elements(type
, GST_RANK_MARGINAL
)))
103 tmp
= gst_element_factory_list_filter(transforms
, src_caps
, GST_PAD_SINK
, FALSE
);
104 gst_plugin_feature_list_free(transforms
);
105 if (!(transforms
= tmp
))
108 tmp
= gst_element_factory_list_filter(transforms
, sink_caps
, GST_PAD_SRC
, FALSE
);
109 gst_plugin_feature_list_free(transforms
);
110 if (!(transforms
= tmp
))
113 transforms
= g_list_sort(transforms
, gst_plugin_feature_rank_compare_func
);
114 for (tmp
= transforms
; tmp
!= NULL
&& element
== NULL
; tmp
= tmp
->next
)
116 name
= gst_plugin_feature_get_name(GST_PLUGIN_FEATURE(tmp
->data
));
117 if (!(element
= gst_element_factory_create(GST_ELEMENT_FACTORY(tmp
->data
), NULL
)))
118 GST_WARNING("Failed to create %s element.", name
);
120 gst_plugin_feature_list_free(transforms
);
125 GST_DEBUG("Created %s element %p.", name
, element
);
129 gchar
*src_str
= gst_caps_to_string(src_caps
), *sink_str
= gst_caps_to_string(sink_caps
);
130 GST_WARNING("Failed to create transform matching caps %s / %s.", src_str
, sink_str
);
138 static bool transform_append_element(struct wg_transform
*transform
, GstElement
*element
,
139 GstElement
**first
, GstElement
**last
)
141 gchar
*name
= gst_element_get_name(element
);
142 bool success
= false;
144 if (!gst_bin_add(GST_BIN(transform
->container
), element
) ||
145 (*last
&& !gst_element_link(*last
, element
)))
147 GST_ERROR("Failed to link %s element.", name
);
151 GST_DEBUG("Linked %s element %p.", name
, element
);
162 NTSTATUS
wg_transform_create(void *args
)
164 struct wg_transform_create_params
*params
= args
;
165 GstCaps
*raw_caps
= NULL
, *src_caps
= NULL
, *sink_caps
= NULL
;
166 struct wg_format output_format
= *params
->output_format
;
167 struct wg_format input_format
= *params
->input_format
;
168 GstElement
*first
= NULL
, *last
= NULL
, *element
;
169 NTSTATUS status
= STATUS_UNSUCCESSFUL
;
170 GstPadTemplate
*template = NULL
;
171 struct wg_transform
*transform
;
172 const gchar
*media_type
;
175 if (!init_gstreamer())
176 return STATUS_UNSUCCESSFUL
;
178 if (!(transform
= calloc(1, sizeof(*transform
))))
179 return STATUS_NO_MEMORY
;
180 if (!(transform
->container
= gst_bin_new("wg_transform")))
182 if (!(transform
->input
= gst_buffer_list_new()))
184 if (!(transform
->output_queue
= gst_atomic_queue_new(8)))
186 transform
->input_max_length
= 1;
188 if (!(src_caps
= wg_format_to_caps(&input_format
)))
190 if (!(template = gst_pad_template_new("src", GST_PAD_SRC
, GST_PAD_ALWAYS
, src_caps
)))
192 transform
->my_src
= gst_pad_new_from_template(template, "src");
193 g_object_unref(template);
194 if (!transform
->my_src
)
197 if (!(sink_caps
= wg_format_to_caps(&output_format
)))
199 if (!(template = gst_pad_template_new("sink", GST_PAD_SINK
, GST_PAD_ALWAYS
, sink_caps
)))
201 transform
->my_sink
= gst_pad_new_from_template(template, "sink");
202 g_object_unref(template);
203 if (!transform
->my_sink
)
206 gst_pad_set_element_private(transform
->my_sink
, transform
);
207 gst_pad_set_chain_function(transform
->my_sink
, transform_sink_chain_cb
);
209 /* Since we append conversion elements, we don't want to filter decoders
210 * based on the actual output caps now. Matching decoders with the
211 * raw output media type should be enough.
213 media_type
= gst_structure_get_name(gst_caps_get_structure(sink_caps
, 0));
214 if (!(raw_caps
= gst_caps_new_empty_simple(media_type
)))
217 switch (input_format
.major_type
)
219 case WG_MAJOR_TYPE_H264
:
220 /* Call of Duty: Black Ops 3 doesn't care about the ProcessInput/ProcessOutput
221 * return values, it calls them in a specific order and and expects the decoder
222 * transform to be able to queue its input buffers. We need to use a buffer list
223 * to match its expectations.
225 transform
->input_max_length
= 16;
227 case WG_MAJOR_TYPE_WMA
:
228 if (!(element
= transform_find_element(GST_ELEMENT_FACTORY_TYPE_DECODER
, src_caps
, raw_caps
))
229 || !transform_append_element(transform
, element
, &first
, &last
))
231 gst_caps_unref(raw_caps
);
236 case WG_MAJOR_TYPE_AUDIO
:
237 case WG_MAJOR_TYPE_VIDEO
:
238 case WG_MAJOR_TYPE_UNKNOWN
:
239 GST_FIXME("Format %u not implemented!", input_format
.major_type
);
240 gst_caps_unref(raw_caps
);
244 gst_caps_unref(raw_caps
);
246 switch (output_format
.major_type
)
248 case WG_MAJOR_TYPE_AUDIO
:
249 /* The MF audio decoder transforms allow decoding to various formats
250 * as well as resampling the audio at the same time, whereas
251 * GStreamer decoder plugins usually only support decoding to a
252 * single format and at the original rate.
254 * The WMA decoder transform also has output samples interleaved on
255 * Windows, whereas GStreamer avdec_wmav2 output uses
256 * non-interleaved format.
258 if (!(element
= create_element("audioconvert", "base"))
259 || !transform_append_element(transform
, element
, &first
, &last
))
261 if (!(element
= create_element("audioresample", "base"))
262 || !transform_append_element(transform
, element
, &first
, &last
))
266 case WG_MAJOR_TYPE_VIDEO
:
269 case WG_MAJOR_TYPE_H264
:
270 case WG_MAJOR_TYPE_WMA
:
271 case WG_MAJOR_TYPE_UNKNOWN
:
272 GST_FIXME("Format %u not implemented!", output_format
.major_type
);
276 if (!(transform
->their_sink
= gst_element_get_static_pad(first
, "sink")))
278 if (!(transform
->their_src
= gst_element_get_static_pad(last
, "src")))
280 if (gst_pad_link(transform
->my_src
, transform
->their_sink
) < 0)
282 if (gst_pad_link(transform
->their_src
, transform
->my_sink
) < 0)
284 if (!gst_pad_set_active(transform
->my_sink
, 1))
286 if (!gst_pad_set_active(transform
->my_src
, 1))
289 gst_element_set_state(transform
->container
, GST_STATE_PAUSED
);
290 if (!gst_element_get_state(transform
->container
, NULL
, NULL
, -1))
293 if (!(event
= gst_event_new_stream_start("stream"))
294 || !gst_pad_push_event(transform
->my_src
, event
))
296 if (!(event
= gst_event_new_caps(src_caps
))
297 || !gst_pad_push_event(transform
->my_src
, event
))
300 /* We need to use GST_FORMAT_TIME here because it's the only format
301 * some elements such avdec_wmav2 correctly support. */
302 gst_segment_init(&transform
->segment
, GST_FORMAT_TIME
);
303 transform
->segment
.start
= 0;
304 transform
->segment
.stop
= -1;
305 if (!(event
= gst_event_new_segment(&transform
->segment
))
306 || !gst_pad_push_event(transform
->my_src
, event
))
309 gst_caps_unref(sink_caps
);
310 gst_caps_unref(src_caps
);
312 GST_INFO("Created winegstreamer transform %p.", transform
);
313 params
->transform
= transform
;
314 return STATUS_SUCCESS
;
317 if (transform
->their_sink
)
318 gst_object_unref(transform
->their_sink
);
319 if (transform
->their_src
)
320 gst_object_unref(transform
->their_src
);
321 if (transform
->my_sink
)
322 gst_object_unref(transform
->my_sink
);
324 gst_caps_unref(sink_caps
);
325 if (transform
->my_src
)
326 gst_object_unref(transform
->my_src
);
328 gst_caps_unref(src_caps
);
329 if (transform
->output_queue
)
330 gst_atomic_queue_unref(transform
->output_queue
);
331 if (transform
->input
)
332 gst_buffer_list_unref(transform
->input
);
333 if (transform
->container
)
335 gst_element_set_state(transform
->container
, GST_STATE_NULL
);
336 gst_object_unref(transform
->container
);
339 GST_ERROR("Failed to create winegstreamer transform.");
343 NTSTATUS
wg_transform_push_data(void *args
)
345 struct wg_transform_push_data_params
*params
= args
;
346 struct wg_transform
*transform
= params
->transform
;
347 struct wg_sample
*sample
= params
->sample
;
351 length
= gst_buffer_list_length(transform
->input
);
352 if (length
>= transform
->input_max_length
)
354 GST_INFO("Refusing %u bytes, %u buffers already queued", sample
->size
, length
);
355 params
->result
= MF_E_NOTACCEPTING
;
356 return STATUS_SUCCESS
;
359 if (!(buffer
= gst_buffer_new_and_alloc(sample
->size
)))
361 GST_ERROR("Failed to allocate input buffer");
362 return STATUS_NO_MEMORY
;
364 gst_buffer_fill(buffer
, 0, sample
->data
, sample
->size
);
365 gst_buffer_list_insert(transform
->input
, -1, buffer
);
367 GST_INFO("Copied %u bytes from sample %p to input buffer list", sample
->size
, sample
);
368 params
->result
= S_OK
;
369 return STATUS_SUCCESS
;
372 static NTSTATUS
read_transform_output_data(GstBuffer
*buffer
, struct wg_sample
*sample
)
376 if (!gst_buffer_map(buffer
, &info
, GST_MAP_READ
))
378 GST_ERROR("Failed to map buffer %p", buffer
);
379 return STATUS_UNSUCCESSFUL
;
382 if (sample
->max_size
>= info
.size
)
383 sample
->size
= info
.size
;
386 sample
->flags
|= WG_SAMPLE_FLAG_INCOMPLETE
;
387 sample
->size
= sample
->max_size
;
390 memcpy(sample
->data
, info
.data
, sample
->size
);
391 gst_buffer_unmap(buffer
, &info
);
392 gst_buffer_resize(buffer
, sample
->size
, -1);
394 GST_INFO("Copied %u bytes, sample %p, flags %#x", sample
->size
, sample
, sample
->flags
);
395 return STATUS_SUCCESS
;
398 NTSTATUS
wg_transform_read_data(void *args
)
400 struct wg_transform_read_data_params
*params
= args
;
401 struct wg_transform
*transform
= params
->transform
;
402 struct wg_sample
*sample
= params
->sample
;
403 GstBufferList
*input
= transform
->input
;
407 if (!gst_buffer_list_length(transform
->input
))
408 GST_DEBUG("Not input buffer queued");
409 else if (!(transform
->input
= gst_buffer_list_new()))
411 GST_ERROR("Failed to allocate new input queue");
412 return STATUS_NO_MEMORY
;
414 else if ((ret
= gst_pad_push_list(transform
->my_src
, input
)))
416 GST_ERROR("Failed to push transform input, error %d", ret
);
417 return STATUS_UNSUCCESSFUL
;
420 if (!transform
->output_buffer
&& !(transform
->output_buffer
= gst_atomic_queue_pop(transform
->output_queue
)))
423 params
->result
= MF_E_TRANSFORM_NEED_MORE_INPUT
;
424 GST_INFO("Cannot read %u bytes, no output available", sample
->max_size
);
425 return STATUS_SUCCESS
;
428 if ((status
= read_transform_output_data(transform
->output_buffer
, sample
)))
434 if (!(sample
->flags
& WG_SAMPLE_FLAG_INCOMPLETE
))
436 gst_buffer_unref(transform
->output_buffer
);
437 transform
->output_buffer
= NULL
;
440 params
->result
= S_OK
;
441 return STATUS_SUCCESS
;