dlls/winegstreamer/wg_transform.c

   1 /*
   2  * GStreamer transform backend
   3  *
   4  * Copyright 2022 Rémi Bernon for CodeWeavers
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
  19  */
  20
  21 #if 0
  22 #pragma makedep unix
  23 #endif
  24
  25 #include "config.h"
  26
  27 #include <assert.h>
  28 #include <stdarg.h>
  29 #include <stdio.h>
  30
  31 #include <gst/gst.h>
  32 #include <gst/video/video.h>
  33 #include <gst/audio/audio.h>
  34
  35 #include "ntstatus.h"
  36 #define WIN32_NO_STATUS
  37 #include "winternl.h"
  38 #include "mferror.h"
  39
  40 #include "unix_private.h"
  41
  42 GST_DEBUG_CATEGORY_EXTERN(wine);
  43 #define GST_CAT_DEFAULT wine
  44
  45 struct wg_transform
  46 {
  47     GstElement *container;
  48     GstPad *my_src, *my_sink;
  49     GstPad *their_sink, *their_src;
  50     GstSegment segment;
  51     GstBufferList *input;
  52     guint input_max_length;
  53     GstAtomicQueue *output_queue;
  54     GstBuffer *output_buffer;
  55 };
  56
  57 static GstFlowReturn transform_sink_chain_cb(GstPad *pad, GstObject *parent, GstBuffer *buffer)
  58 {
  59     struct wg_transform *transform = gst_pad_get_element_private(pad);
  60
  61     GST_LOG("transform %p, buffer %p.", transform, buffer);
  62
  63     gst_atomic_queue_push(transform->output_queue, buffer);
  64
  65     return GST_FLOW_OK;
  66 }
  67
  68 NTSTATUS wg_transform_destroy(void *args)
  69 {
  70     struct wg_transform *transform = args;
  71     GstBuffer *buffer;
  72
  73     if (transform->input)
  74         gst_buffer_list_unref(transform->input);
  75
  76     gst_element_set_state(transform->container, GST_STATE_NULL);
  77
  78     if (transform->output_buffer)
  79         gst_buffer_unref(transform->output_buffer);
  80     while ((buffer = gst_atomic_queue_pop(transform->output_queue)))
  81         gst_buffer_unref(buffer);
  82
  83     g_object_unref(transform->their_sink);
  84     g_object_unref(transform->their_src);
  85     g_object_unref(transform->container);
  86     g_object_unref(transform->my_sink);
  87     g_object_unref(transform->my_src);
  88     gst_atomic_queue_unref(transform->output_queue);
  89     free(transform);
  90
  91     return STATUS_SUCCESS;
  92 }
  93
  94 static GstElement *transform_find_element(GstElementFactoryListType type, GstCaps *src_caps, GstCaps *sink_caps)
  95 {
  96     GstElement *element = NULL;
  97     GList *tmp, *transforms;
  98     const gchar *name;
  99
 100     if (!(transforms = gst_element_factory_list_get_elements(type, GST_RANK_MARGINAL)))
 101         goto done;
 102
 103     tmp = gst_element_factory_list_filter(transforms, src_caps, GST_PAD_SINK, FALSE);
 104     gst_plugin_feature_list_free(transforms);
 105     if (!(transforms = tmp))
 106         goto done;
 107
 108     tmp = gst_element_factory_list_filter(transforms, sink_caps, GST_PAD_SRC, FALSE);
 109     gst_plugin_feature_list_free(transforms);
 110     if (!(transforms = tmp))
 111         goto done;
 112
 113     transforms = g_list_sort(transforms, gst_plugin_feature_rank_compare_func);
 114     for (tmp = transforms; tmp != NULL && element == NULL; tmp = tmp->next)
 115     {
 116         name = gst_plugin_feature_get_name(GST_PLUGIN_FEATURE(tmp->data));
 117         if (!(element = gst_element_factory_create(GST_ELEMENT_FACTORY(tmp->data), NULL)))
 118             GST_WARNING("Failed to create %s element.", name);
 119     }
 120     gst_plugin_feature_list_free(transforms);
 121
 122 done:
 123     if (element)
 124     {
 125         GST_DEBUG("Created %s element %p.", name, element);
 126     }
 127     else
 128     {
 129         gchar *src_str = gst_caps_to_string(src_caps), *sink_str = gst_caps_to_string(sink_caps);
 130         GST_WARNING("Failed to create transform matching caps %s / %s.", src_str, sink_str);
 131         g_free(sink_str);
 132         g_free(src_str);
 133     }
 134
 135     return element;
 136 }
 137
 138 static bool transform_append_element(struct wg_transform *transform, GstElement *element,
 139         GstElement **first, GstElement **last)
 140 {
 141     gchar *name = gst_element_get_name(element);
 142     bool success = false;
 143
 144     if (!gst_bin_add(GST_BIN(transform->container), element) ||
 145             (*last && !gst_element_link(*last, element)))
 146     {
 147         GST_ERROR("Failed to link %s element.", name);
 148     }
 149     else
 150     {
 151         GST_DEBUG("Linked %s element %p.", name, element);
 152         if (!*first)
 153             *first = element;
 154         *last = element;
 155         success = true;
 156     }
 157
 158     g_free(name);
 159     return success;
 160 }
 161
 162 NTSTATUS wg_transform_create(void *args)
 163 {
 164     struct wg_transform_create_params *params = args;
 165     GstCaps *raw_caps = NULL, *src_caps = NULL, *sink_caps = NULL;
 166     struct wg_format output_format = *params->output_format;
 167     struct wg_format input_format = *params->input_format;
 168     GstElement *first = NULL, *last = NULL, *element;
 169     NTSTATUS status = STATUS_UNSUCCESSFUL;
 170     GstPadTemplate *template = NULL;
 171     struct wg_transform *transform;
 172     const gchar *media_type;
 173     GstEvent *event;
 174
 175     if (!init_gstreamer())
 176         return STATUS_UNSUCCESSFUL;
 177
 178     if (!(transform = calloc(1, sizeof(*transform))))
 179         return STATUS_NO_MEMORY;
 180     if (!(transform->container = gst_bin_new("wg_transform")))
 181         goto out;
 182     if (!(transform->input = gst_buffer_list_new()))
 183         goto out;
 184     if (!(transform->output_queue = gst_atomic_queue_new(8)))
 185         goto out;
 186     transform->input_max_length = 1;
 187
 188     if (!(src_caps = wg_format_to_caps(&input_format)))
 189         goto out;
 190     if (!(template = gst_pad_template_new("src", GST_PAD_SRC, GST_PAD_ALWAYS, src_caps)))
 191         goto out;
 192     transform->my_src = gst_pad_new_from_template(template, "src");
 193     g_object_unref(template);
 194     if (!transform->my_src)
 195         goto out;
 196
 197     if (!(sink_caps = wg_format_to_caps(&output_format)))
 198         goto out;
 199     if (!(template = gst_pad_template_new("sink", GST_PAD_SINK, GST_PAD_ALWAYS, sink_caps)))
 200         goto out;
 201     transform->my_sink = gst_pad_new_from_template(template, "sink");
 202     g_object_unref(template);
 203     if (!transform->my_sink)
 204         goto out;
 205
 206     gst_pad_set_element_private(transform->my_sink, transform);
 207     gst_pad_set_chain_function(transform->my_sink, transform_sink_chain_cb);
 208
 209     /* Since we append conversion elements, we don't want to filter decoders
 210      * based on the actual output caps now. Matching decoders with the
 211      * raw output media type should be enough.
 212      */
 213     media_type = gst_structure_get_name(gst_caps_get_structure(sink_caps, 0));
 214     if (!(raw_caps = gst_caps_new_empty_simple(media_type)))
 215         goto out;
 216
 217     switch (input_format.major_type)
 218     {
 219         case WG_MAJOR_TYPE_H264:
 220             /* Call of Duty: Black Ops 3 doesn't care about the ProcessInput/ProcessOutput
 221              * return values, it calls them in a specific order and and expects the decoder
 222              * transform to be able to queue its input buffers. We need to use a buffer list
 223              * to match its expectations.
 224              */
 225             transform->input_max_length = 16;
 226             /* fallthrough */
 227         case WG_MAJOR_TYPE_WMA:
 228             if (!(element = transform_find_element(GST_ELEMENT_FACTORY_TYPE_DECODER, src_caps, raw_caps))
 229                     || !transform_append_element(transform, element, &first, &last))
 230             {
 231                 gst_caps_unref(raw_caps);
 232                 goto out;
 233             }
 234             break;
 235
 236         case WG_MAJOR_TYPE_AUDIO:
 237         case WG_MAJOR_TYPE_VIDEO:
 238         case WG_MAJOR_TYPE_UNKNOWN:
 239             GST_FIXME("Format %u not implemented!", input_format.major_type);
 240             gst_caps_unref(raw_caps);
 241             goto out;
 242     }
 243
 244     gst_caps_unref(raw_caps);
 245
 246     switch (output_format.major_type)
 247     {
 248         case WG_MAJOR_TYPE_AUDIO:
 249             /* The MF audio decoder transforms allow decoding to various formats
 250              * as well as resampling the audio at the same time, whereas
 251              * GStreamer decoder plugins usually only support decoding to a
 252              * single format and at the original rate.
 253              *
 254              * The WMA decoder transform also has output samples interleaved on
 255              * Windows, whereas GStreamer avdec_wmav2 output uses
 256              * non-interleaved format.
 257              */
 258             if (!(element = create_element("audioconvert", "base"))
 259                     || !transform_append_element(transform, element, &first, &last))
 260                 goto out;
 261             if (!(element = create_element("audioresample", "base"))
 262                     || !transform_append_element(transform, element, &first, &last))
 263                 goto out;
 264             break;
 265
 266         case WG_MAJOR_TYPE_VIDEO:
 267             break;
 268
 269         case WG_MAJOR_TYPE_H264:
 270         case WG_MAJOR_TYPE_WMA:
 271         case WG_MAJOR_TYPE_UNKNOWN:
 272             GST_FIXME("Format %u not implemented!", output_format.major_type);
 273             goto out;
 274     }
 275
 276     if (!(transform->their_sink = gst_element_get_static_pad(first, "sink")))
 277         goto out;
 278     if (!(transform->their_src = gst_element_get_static_pad(last, "src")))
 279         goto out;
 280     if (gst_pad_link(transform->my_src, transform->their_sink) < 0)
 281         goto out;
 282     if (gst_pad_link(transform->their_src, transform->my_sink) < 0)
 283         goto out;
 284     if (!gst_pad_set_active(transform->my_sink, 1))
 285         goto out;
 286     if (!gst_pad_set_active(transform->my_src, 1))
 287         goto out;
 288
 289     gst_element_set_state(transform->container, GST_STATE_PAUSED);
 290     if (!gst_element_get_state(transform->container, NULL, NULL, -1))
 291         goto out;
 292
 293     if (!(event = gst_event_new_stream_start("stream"))
 294             || !gst_pad_push_event(transform->my_src, event))
 295         goto out;
 296     if (!(event = gst_event_new_caps(src_caps))
 297             || !gst_pad_push_event(transform->my_src, event))
 298         goto out;
 299
 300     /* We need to use GST_FORMAT_TIME here because it's the only format
 301      * some elements such avdec_wmav2 correctly support. */
 302     gst_segment_init(&transform->segment, GST_FORMAT_TIME);
 303     transform->segment.start = 0;
 304     transform->segment.stop = -1;
 305     if (!(event = gst_event_new_segment(&transform->segment))
 306             || !gst_pad_push_event(transform->my_src, event))
 307         goto out;
 308
 309     gst_caps_unref(sink_caps);
 310     gst_caps_unref(src_caps);
 311
 312     GST_INFO("Created winegstreamer transform %p.", transform);
 313     params->transform = transform;
 314     return STATUS_SUCCESS;
 315
 316 out:
 317     if (transform->their_sink)
 318         gst_object_unref(transform->their_sink);
 319     if (transform->their_src)
 320         gst_object_unref(transform->their_src);
 321     if (transform->my_sink)
 322         gst_object_unref(transform->my_sink);
 323     if (sink_caps)
 324         gst_caps_unref(sink_caps);
 325     if (transform->my_src)
 326         gst_object_unref(transform->my_src);
 327     if (src_caps)
 328         gst_caps_unref(src_caps);
 329     if (transform->output_queue)
 330         gst_atomic_queue_unref(transform->output_queue);
 331     if (transform->input)
 332         gst_buffer_list_unref(transform->input);
 333     if (transform->container)
 334     {
 335         gst_element_set_state(transform->container, GST_STATE_NULL);
 336         gst_object_unref(transform->container);
 337     }
 338     free(transform);
 339     GST_ERROR("Failed to create winegstreamer transform.");
 340     return status;
 341 }
 342
 343 NTSTATUS wg_transform_push_data(void *args)
 344 {
 345     struct wg_transform_push_data_params *params = args;
 346     struct wg_transform *transform = params->transform;
 347     struct wg_sample *sample = params->sample;
 348     GstBuffer *buffer;
 349     guint length;
 350
 351     length = gst_buffer_list_length(transform->input);
 352     if (length >= transform->input_max_length)
 353     {
 354         GST_INFO("Refusing %u bytes, %u buffers already queued", sample->size, length);
 355         params->result = MF_E_NOTACCEPTING;
 356         return STATUS_SUCCESS;
 357     }
 358
 359     if (!(buffer = gst_buffer_new_and_alloc(sample->size)))
 360     {
 361         GST_ERROR("Failed to allocate input buffer");
 362         return STATUS_NO_MEMORY;
 363     }
 364     gst_buffer_fill(buffer, 0, sample->data, sample->size);
 365     gst_buffer_list_insert(transform->input, -1, buffer);
 366
 367     GST_INFO("Copied %u bytes from sample %p to input buffer list", sample->size, sample);
 368     params->result = S_OK;
 369     return STATUS_SUCCESS;
 370 }
 371
 372 static NTSTATUS read_transform_output_data(GstBuffer *buffer, struct wg_sample *sample)
 373 {
 374     GstMapInfo info;
 375
 376     if (!gst_buffer_map(buffer, &info, GST_MAP_READ))
 377     {
 378         GST_ERROR("Failed to map buffer %p", buffer);
 379         return STATUS_UNSUCCESSFUL;
 380     }
 381
 382     if (sample->max_size >= info.size)
 383         sample->size = info.size;
 384     else
 385     {
 386         sample->flags |= WG_SAMPLE_FLAG_INCOMPLETE;
 387         sample->size = sample->max_size;
 388     }
 389
 390     memcpy(sample->data, info.data, sample->size);
 391     gst_buffer_unmap(buffer, &info);
 392     gst_buffer_resize(buffer, sample->size, -1);
 393
 394     GST_INFO("Copied %u bytes, sample %p, flags %#x", sample->size, sample, sample->flags);
 395     return STATUS_SUCCESS;
 396 }
 397
 398 NTSTATUS wg_transform_read_data(void *args)
 399 {
 400     struct wg_transform_read_data_params *params = args;
 401     struct wg_transform *transform = params->transform;
 402     struct wg_sample *sample = params->sample;
 403     GstBufferList *input = transform->input;
 404     GstFlowReturn ret;
 405     NTSTATUS status;
 406
 407     if (!gst_buffer_list_length(transform->input))
 408         GST_DEBUG("Not input buffer queued");
 409     else if (!(transform->input = gst_buffer_list_new()))
 410     {
 411         GST_ERROR("Failed to allocate new input queue");
 412         return STATUS_NO_MEMORY;
 413     }
 414     else if ((ret = gst_pad_push_list(transform->my_src, input)))
 415     {
 416         GST_ERROR("Failed to push transform input, error %d", ret);
 417         return STATUS_UNSUCCESSFUL;
 418     }
 419
 420     if (!transform->output_buffer && !(transform->output_buffer = gst_atomic_queue_pop(transform->output_queue)))
 421     {
 422         sample->size = 0;
 423         params->result = MF_E_TRANSFORM_NEED_MORE_INPUT;
 424         GST_INFO("Cannot read %u bytes, no output available", sample->max_size);
 425         return STATUS_SUCCESS;
 426     }
 427
 428     if ((status = read_transform_output_data(transform->output_buffer, sample)))
 429     {
 430         sample->size = 0;
 431         return status;
 432     }
 433
 434     if (!(sample->flags & WG_SAMPLE_FLAG_INCOMPLETE))
 435     {
 436         gst_buffer_unref(transform->output_buffer);
 437         transform->output_buffer = NULL;
 438     }
 439
 440     params->result = S_OK;
 441     return STATUS_SUCCESS;
 442 }