dlls/winegstreamer/wg_transform.c

   1 /*
   2  * GStreamer transform backend
   3  *
   4  * Copyright 2022 Rémi Bernon for CodeWeavers
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
  19  */
  20
  21 #if 0
  22 #pragma makedep unix
  23 #endif
  24
  25 #include "config.h"
  26
  27 #include <assert.h>
  28 #include <stdarg.h>
  29 #include <stdio.h>
  30
  31 #include <gst/gst.h>
  32 #include <gst/video/video.h>
  33 #include <gst/audio/audio.h>
  34
  35 #include "ntstatus.h"
  36 #define WIN32_NO_STATUS
  37 #include "winternl.h"
  38 #include "mferror.h"
  39
  40 #include "unix_private.h"
  41
  42 #define GST_SAMPLE_FLAG_WG_CAPS_CHANGED (GST_MINI_OBJECT_FLAG_LAST << 0)
  43
  44 struct wg_transform
  45 {
  46     struct wg_transform_attrs attrs;
  47
  48     GstElement *container;
  49     GstAllocator *allocator;
  50     GstPad *my_src, *my_sink;
  51     GstSegment segment;
  52     GstQuery *drain_query;
  53
  54     GstAtomicQueue *input_queue;
  55
  56     bool input_is_flipped;
  57     GstElement *video_flip;
  58
  59     struct wg_format output_format;
  60     GstAtomicQueue *output_queue;
  61     GstSample *output_sample;
  62     bool output_caps_changed;
  63     GstCaps *output_caps;
  64 };
  65
  66 static struct wg_transform *get_transform(wg_transform_t trans)
  67 {
  68     return (struct wg_transform *)(ULONG_PTR)trans;
  69 }
  70
  71 static void align_video_info_planes(gsize plane_align, GstVideoInfo *info, GstVideoAlignment *align)
  72 {
  73     gst_video_alignment_reset(align);
  74
  75     align->padding_right = ((plane_align + 1) - (info->width & plane_align)) & plane_align;
  76     align->padding_bottom = ((plane_align + 1) - (info->height & plane_align)) & plane_align;
  77     align->stride_align[0] = plane_align;
  78     align->stride_align[1] = plane_align;
  79     align->stride_align[2] = plane_align;
  80     align->stride_align[3] = plane_align;
  81
  82     gst_video_info_align(info, align);
  83 }
  84
  85 static GstFlowReturn transform_sink_chain_cb(GstPad *pad, GstObject *parent, GstBuffer *buffer)
  86 {
  87     struct wg_transform *transform = gst_pad_get_element_private(pad);
  88     GstSample *sample;
  89
  90     GST_LOG("transform %p, buffer %p.", transform, buffer);
  91
  92     if (!(sample = gst_sample_new(buffer, transform->output_caps, NULL, NULL)))
  93     {
  94         GST_ERROR("Failed to allocate transform %p output sample.", transform);
  95         gst_buffer_unref(buffer);
  96         return GST_FLOW_ERROR;
  97     }
  98
  99     if (transform->output_caps_changed)
 100         GST_MINI_OBJECT_FLAG_SET(sample, GST_SAMPLE_FLAG_WG_CAPS_CHANGED);
 101     transform->output_caps_changed = false;
 102
 103     gst_atomic_queue_push(transform->output_queue, sample);
 104     gst_buffer_unref(buffer);
 105     return GST_FLOW_OK;
 106 }
 107
 108 static gboolean transform_src_query_latency(struct wg_transform *transform, GstQuery *query)
 109 {
 110     GST_LOG("transform %p, query %p", transform, query);
 111     gst_query_set_latency(query, transform->attrs.low_latency, 0, 0);
 112     return true;
 113 }
 114
 115 static gboolean transform_src_query_cb(GstPad *pad, GstObject *parent, GstQuery *query)
 116 {
 117     struct wg_transform *transform = gst_pad_get_element_private(pad);
 118
 119     switch (query->type)
 120     {
 121     case GST_QUERY_LATENCY:
 122         return transform_src_query_latency(transform, query);
 123     default:
 124         return gst_pad_query_default(pad, parent, query);
 125     }
 126 }
 127
 128 static gboolean transform_sink_query_cb(GstPad *pad, GstObject *parent, GstQuery *query)
 129 {
 130     struct wg_transform *transform = gst_pad_get_element_private(pad);
 131
 132     GST_LOG("transform %p, type \"%s\".", transform, gst_query_type_get_name(query->type));
 133
 134     switch (query->type)
 135     {
 136         case GST_QUERY_ALLOCATION:
 137         {
 138             gsize plane_align = transform->attrs.output_plane_align;
 139             GstStructure *config, *params;
 140             GstVideoAlignment align;
 141             gboolean needs_pool;
 142             GstBufferPool *pool;
 143             GstVideoInfo info;
 144             GstCaps *caps;
 145
 146             gst_query_parse_allocation(query, &caps, &needs_pool);
 147             if (stream_type_from_caps(caps) != GST_STREAM_TYPE_VIDEO || !needs_pool)
 148                 break;
 149
 150             if (!gst_video_info_from_caps(&info, caps)
 151                     || !(pool = gst_video_buffer_pool_new()))
 152                 break;
 153
 154             align_video_info_planes(plane_align, &info, &align);
 155
 156             if ((params = gst_structure_new("video-meta",
 157                     "padding-top", G_TYPE_UINT, align.padding_top,
 158                     "padding-bottom", G_TYPE_UINT, align.padding_bottom,
 159                     "padding-left", G_TYPE_UINT, align.padding_left,
 160                     "padding-right", G_TYPE_UINT, align.padding_right,
 161                     NULL)))
 162             {
 163                 gst_query_add_allocation_meta(query, GST_VIDEO_META_API_TYPE, params);
 164                 gst_structure_free(params);
 165             }
 166
 167             if (!(config = gst_buffer_pool_get_config(pool)))
 168                 GST_ERROR("Failed to get pool %p config.", pool);
 169             else
 170             {
 171                 gst_buffer_pool_config_add_option(config, GST_BUFFER_POOL_OPTION_VIDEO_META);
 172                 gst_buffer_pool_config_add_option(config, GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT);
 173                 gst_buffer_pool_config_set_video_alignment(config, &align);
 174
 175                 gst_buffer_pool_config_set_params(config, caps,
 176                         info.size, 0, 0);
 177                 gst_buffer_pool_config_set_allocator(config, transform->allocator, NULL);
 178                 if (!gst_buffer_pool_set_config(pool, config))
 179                     GST_ERROR("Failed to set pool %p config.", pool);
 180             }
 181
 182             /* Prevent pool reconfiguration, we don't want another alignment. */
 183             if (!gst_buffer_pool_set_active(pool, true))
 184                 GST_ERROR("Pool %p failed to activate.", pool);
 185
 186             gst_query_add_allocation_pool(query, pool, info.size, 0, 0);
 187             gst_query_add_allocation_param(query, transform->allocator, NULL);
 188
 189             GST_INFO("Proposing pool %p, buffer size %#zx, allocator %p, for query %p.",
 190                     pool, info.size, transform->allocator, query);
 191
 192             g_object_unref(pool);
 193             return true;
 194         }
 195
 196         case GST_QUERY_CAPS:
 197         {
 198             GstCaps *caps, *filter, *temp;
 199             gchar *str;
 200
 201             gst_query_parse_caps(query, &filter);
 202             if (!(caps = wg_format_to_caps(&transform->output_format)))
 203                 break;
 204
 205             if (filter)
 206             {
 207                 temp = gst_caps_intersect(caps, filter);
 208                 gst_caps_unref(caps);
 209                 caps = temp;
 210             }
 211
 212             str = gst_caps_to_string(caps);
 213             GST_INFO("Returning caps %s", str);
 214             g_free(str);
 215
 216             gst_query_set_caps_result(query, caps);
 217             gst_caps_unref(caps);
 218             return true;
 219         }
 220
 221         default:
 222             GST_WARNING("Ignoring \"%s\" query.", gst_query_type_get_name(query->type));
 223             break;
 224     }
 225
 226     return gst_pad_query_default(pad, parent, query);
 227 }
 228
 229 static gboolean transform_sink_event_cb(GstPad *pad, GstObject *parent, GstEvent *event)
 230 {
 231     struct wg_transform *transform = gst_pad_get_element_private(pad);
 232
 233     GST_LOG("transform %p, type \"%s\".", transform, GST_EVENT_TYPE_NAME(event));
 234
 235     switch (event->type)
 236     {
 237         case GST_EVENT_CAPS:
 238         {
 239             GstCaps *caps;
 240
 241             gst_event_parse_caps(event, &caps);
 242
 243             transform->output_caps_changed = transform->output_caps_changed
 244                     || !gst_caps_is_always_compatible(transform->output_caps, caps);
 245
 246             gst_caps_unref(transform->output_caps);
 247             transform->output_caps = gst_caps_ref(caps);
 248             break;
 249         }
 250         default:
 251             GST_WARNING("Ignoring \"%s\" event.", GST_EVENT_TYPE_NAME(event));
 252             break;
 253     }
 254
 255     gst_event_unref(event);
 256     return TRUE;
 257 }
 258
 259 NTSTATUS wg_transform_destroy(void *args)
 260 {
 261     struct wg_transform *transform = get_transform(*(wg_transform_t *)args);
 262     GstSample *sample;
 263     GstBuffer *buffer;
 264
 265     while ((buffer = gst_atomic_queue_pop(transform->input_queue)))
 266         gst_buffer_unref(buffer);
 267     gst_atomic_queue_unref(transform->input_queue);
 268
 269     gst_element_set_state(transform->container, GST_STATE_NULL);
 270
 271     if (transform->output_sample)
 272         gst_sample_unref(transform->output_sample);
 273     while ((sample = gst_atomic_queue_pop(transform->output_queue)))
 274         gst_sample_unref(sample);
 275
 276     wg_allocator_destroy(transform->allocator);
 277     g_object_unref(transform->container);
 278     g_object_unref(transform->my_sink);
 279     g_object_unref(transform->my_src);
 280     gst_query_unref(transform->drain_query);
 281     gst_caps_unref(transform->output_caps);
 282     gst_atomic_queue_unref(transform->output_queue);
 283     free(transform);
 284
 285     return STATUS_SUCCESS;
 286 }
 287
 288 static bool wg_format_video_is_flipped(const struct wg_format *format)
 289 {
 290     return format->major_type == WG_MAJOR_TYPE_VIDEO && (format->u.video.height < 0);
 291 }
 292
 293 NTSTATUS wg_transform_create(void *args)
 294 {
 295     struct wg_transform_create_params *params = args;
 296     struct wg_format output_format = *params->output_format;
 297     struct wg_format input_format = *params->input_format;
 298     GstElement *first = NULL, *last = NULL, *element;
 299     GstCaps *raw_caps = NULL, *src_caps = NULL;
 300     NTSTATUS status = STATUS_UNSUCCESSFUL;
 301     GstPadTemplate *template = NULL;
 302     struct wg_transform *transform;
 303     const gchar *media_type;
 304     GstEvent *event;
 305
 306     if (!(transform = calloc(1, sizeof(*transform))))
 307         return STATUS_NO_MEMORY;
 308     if (!(transform->container = gst_bin_new("wg_transform")))
 309         goto out;
 310     if (!(transform->input_queue = gst_atomic_queue_new(8)))
 311         goto out;
 312     if (!(transform->output_queue = gst_atomic_queue_new(8)))
 313         goto out;
 314     if (!(transform->drain_query = gst_query_new_drain()))
 315         goto out;
 316     if (!(transform->allocator = wg_allocator_create()))
 317         goto out;
 318     transform->attrs = *params->attrs;
 319     transform->output_format = output_format;
 320
 321     if (!(src_caps = wg_format_to_caps(&input_format)))
 322         goto out;
 323     if (!(template = gst_pad_template_new("src", GST_PAD_SRC, GST_PAD_ALWAYS, src_caps)))
 324         goto out;
 325     transform->my_src = gst_pad_new_from_template(template, "src");
 326     g_object_unref(template);
 327     if (!transform->my_src)
 328         goto out;
 329
 330     gst_pad_set_element_private(transform->my_src, transform);
 331     gst_pad_set_query_function(transform->my_src, transform_src_query_cb);
 332
 333     if (!(transform->output_caps = wg_format_to_caps(&output_format)))
 334         goto out;
 335     if (!(template = gst_pad_template_new("sink", GST_PAD_SINK, GST_PAD_ALWAYS, transform->output_caps)))
 336         goto out;
 337     transform->my_sink = gst_pad_new_from_template(template, "sink");
 338     g_object_unref(template);
 339     if (!transform->my_sink)
 340         goto out;
 341
 342     gst_pad_set_element_private(transform->my_sink, transform);
 343     gst_pad_set_event_function(transform->my_sink, transform_sink_event_cb);
 344     gst_pad_set_query_function(transform->my_sink, transform_sink_query_cb);
 345     gst_pad_set_chain_function(transform->my_sink, transform_sink_chain_cb);
 346
 347     /* Since we append conversion elements, we don't want to filter decoders
 348      * based on the actual output caps now. Matching decoders with the
 349      * raw output media type should be enough.
 350      */
 351     media_type = gst_structure_get_name(gst_caps_get_structure(transform->output_caps, 0));
 352     if (!(raw_caps = gst_caps_new_empty_simple(media_type)))
 353         goto out;
 354
 355     switch (input_format.major_type)
 356     {
 357         case WG_MAJOR_TYPE_VIDEO_H264:
 358         case WG_MAJOR_TYPE_AUDIO_MPEG1:
 359         case WG_MAJOR_TYPE_AUDIO_MPEG4:
 360         case WG_MAJOR_TYPE_AUDIO_WMA:
 361         case WG_MAJOR_TYPE_VIDEO_CINEPAK:
 362         case WG_MAJOR_TYPE_VIDEO_INDEO:
 363         case WG_MAJOR_TYPE_VIDEO_WMV:
 364             if (!(element = find_element(GST_ELEMENT_FACTORY_TYPE_DECODER, src_caps, raw_caps))
 365                     || !append_element(transform->container, element, &first, &last))
 366             {
 367                 gst_caps_unref(raw_caps);
 368                 goto out;
 369             }
 370             break;
 371
 372         case WG_MAJOR_TYPE_AUDIO:
 373         case WG_MAJOR_TYPE_VIDEO:
 374             break;
 375         case WG_MAJOR_TYPE_UNKNOWN:
 376             GST_FIXME("Format %u not implemented!", input_format.major_type);
 377             gst_caps_unref(raw_caps);
 378             goto out;
 379     }
 380
 381     gst_caps_unref(raw_caps);
 382
 383     switch (output_format.major_type)
 384     {
 385         case WG_MAJOR_TYPE_AUDIO:
 386             /* The MF audio decoder transforms allow decoding to various formats
 387              * as well as resampling the audio at the same time, whereas
 388              * GStreamer decoder plugins usually only support decoding to a
 389              * single format and at the original rate.
 390              *
 391              * The WMA decoder transform also has output samples interleaved on
 392              * Windows, whereas GStreamer avdec_wmav2 output uses
 393              * non-interleaved format.
 394              */
 395             if (!(element = create_element("audioconvert", "base"))
 396                     || !append_element(transform->container, element, &first, &last))
 397                 goto out;
 398             if (!(element = create_element("audioresample", "base"))
 399                     || !append_element(transform->container, element, &first, &last))
 400                 goto out;
 401             break;
 402
 403         case WG_MAJOR_TYPE_VIDEO:
 404             if (!(element = create_element("videoconvert", "base"))
 405                     || !append_element(transform->container, element, &first, &last))
 406                 goto out;
 407             if (!(transform->video_flip = create_element("videoflip", "base"))
 408                     || !append_element(transform->container, transform->video_flip, &first, &last))
 409                 goto out;
 410             transform->input_is_flipped = wg_format_video_is_flipped(&input_format);
 411             if (transform->input_is_flipped != wg_format_video_is_flipped(&output_format))
 412                 gst_util_set_object_arg(G_OBJECT(transform->video_flip), "method", "vertical-flip");
 413             if (!(element = create_element("videoconvert", "base"))
 414                     || !append_element(transform->container, element, &first, &last))
 415                 goto out;
 416             /* Let GStreamer choose a default number of threads. */
 417             gst_util_set_object_arg(G_OBJECT(element), "n-threads", "0");
 418             break;
 419
 420         case WG_MAJOR_TYPE_UNKNOWN:
 421         case WG_MAJOR_TYPE_AUDIO_MPEG1:
 422         case WG_MAJOR_TYPE_AUDIO_MPEG4:
 423         case WG_MAJOR_TYPE_AUDIO_WMA:
 424         case WG_MAJOR_TYPE_VIDEO_CINEPAK:
 425         case WG_MAJOR_TYPE_VIDEO_H264:
 426         case WG_MAJOR_TYPE_VIDEO_INDEO:
 427         case WG_MAJOR_TYPE_VIDEO_WMV:
 428             GST_FIXME("Format %u not implemented!", output_format.major_type);
 429             goto out;
 430     }
 431
 432     if (!link_src_to_element(transform->my_src, first))
 433         goto out;
 434     if (!link_element_to_sink(last, transform->my_sink))
 435         goto out;
 436     if (!gst_pad_set_active(transform->my_sink, 1))
 437         goto out;
 438     if (!gst_pad_set_active(transform->my_src, 1))
 439         goto out;
 440
 441     gst_element_set_state(transform->container, GST_STATE_PAUSED);
 442     if (!gst_element_get_state(transform->container, NULL, NULL, -1))
 443         goto out;
 444
 445     if (!(event = gst_event_new_stream_start("stream"))
 446             || !gst_pad_push_event(transform->my_src, event))
 447         goto out;
 448     if (!(event = gst_event_new_caps(src_caps))
 449             || !gst_pad_push_event(transform->my_src, event))
 450         goto out;
 451
 452     /* We need to use GST_FORMAT_TIME here because it's the only format
 453      * some elements such avdec_wmav2 correctly support. */
 454     gst_segment_init(&transform->segment, GST_FORMAT_TIME);
 455     transform->segment.start = 0;
 456     transform->segment.stop = -1;
 457     if (!(event = gst_event_new_segment(&transform->segment))
 458             || !gst_pad_push_event(transform->my_src, event))
 459         goto out;
 460
 461     gst_caps_unref(src_caps);
 462
 463     GST_INFO("Created winegstreamer transform %p.", transform);
 464     params->transform = (wg_transform_t)(ULONG_PTR)transform;
 465     return STATUS_SUCCESS;
 466
 467 out:
 468     if (transform->my_sink)
 469         gst_object_unref(transform->my_sink);
 470     if (transform->output_caps)
 471         gst_caps_unref(transform->output_caps);
 472     if (transform->my_src)
 473         gst_object_unref(transform->my_src);
 474     if (src_caps)
 475         gst_caps_unref(src_caps);
 476     if (transform->allocator)
 477         wg_allocator_destroy(transform->allocator);
 478     if (transform->drain_query)
 479         gst_query_unref(transform->drain_query);
 480     if (transform->output_queue)
 481         gst_atomic_queue_unref(transform->output_queue);
 482     if (transform->input_queue)
 483         gst_atomic_queue_unref(transform->input_queue);
 484     if (transform->container)
 485     {
 486         gst_element_set_state(transform->container, GST_STATE_NULL);
 487         gst_object_unref(transform->container);
 488     }
 489     free(transform);
 490     GST_ERROR("Failed to create winegstreamer transform.");
 491     return status;
 492 }
 493
 494 NTSTATUS wg_transform_set_output_format(void *args)
 495 {
 496     struct wg_transform_set_output_format_params *params = args;
 497     struct wg_transform *transform = get_transform(params->transform);
 498     const struct wg_format *format = params->format;
 499     GstSample *sample;
 500     GstCaps *caps;
 501     gchar *str;
 502
 503     if (!(caps = wg_format_to_caps(format)))
 504     {
 505         GST_ERROR("Failed to convert format %p to caps.", format);
 506         return STATUS_UNSUCCESSFUL;
 507     }
 508     transform->output_format = *format;
 509
 510     if (gst_caps_is_always_compatible(transform->output_caps, caps))
 511     {
 512         gst_caps_unref(caps);
 513         return STATUS_SUCCESS;
 514     }
 515
 516     if (!gst_pad_peer_query(transform->my_src, transform->drain_query))
 517     {
 518         GST_ERROR("Failed to drain transform %p.", transform);
 519         return STATUS_UNSUCCESSFUL;
 520     }
 521
 522     gst_caps_unref(transform->output_caps);
 523     transform->output_caps = caps;
 524
 525     if (transform->video_flip)
 526     {
 527         const char *value;
 528         if (transform->input_is_flipped != wg_format_video_is_flipped(format))
 529             value = "vertical-flip";
 530         else
 531             value = "none";
 532         gst_util_set_object_arg(G_OBJECT(transform->video_flip), "method", value);
 533     }
 534     if (!gst_pad_push_event(transform->my_sink, gst_event_new_reconfigure()))
 535     {
 536         GST_ERROR("Failed to reconfigure transform %p.", transform);
 537         return STATUS_UNSUCCESSFUL;
 538     }
 539
 540     str = gst_caps_to_string(caps);
 541     GST_INFO("Configured new caps %s.", str);
 542     g_free(str);
 543
 544     /* Ideally and to be fully compatible with native transform, the queued
 545      * output buffers will need to be converted to the new output format and
 546      * kept queued.
 547      */
 548     if (transform->output_sample)
 549         gst_sample_unref(transform->output_sample);
 550     while ((sample = gst_atomic_queue_pop(transform->output_queue)))
 551         gst_sample_unref(sample);
 552     transform->output_sample = NULL;
 553
 554     return STATUS_SUCCESS;
 555 }
 556
 557 static void wg_sample_free_notify(void *arg)
 558 {
 559     struct wg_sample *sample = arg;
 560     GST_DEBUG("Releasing wg_sample %p", sample);
 561     InterlockedDecrement(&sample->refcount);
 562 }
 563
 564 NTSTATUS wg_transform_push_data(void *args)
 565 {
 566     struct wg_transform_push_data_params *params = args;
 567     struct wg_transform *transform = get_transform(params->transform);
 568     struct wg_sample *sample = params->sample;
 569     GstBuffer *buffer;
 570     guint length;
 571
 572     length = gst_atomic_queue_length(transform->input_queue);
 573     if (length >= transform->attrs.input_queue_length + 1)
 574     {
 575         GST_INFO("Refusing %u bytes, %u buffers already queued", sample->size, length);
 576         params->result = MF_E_NOTACCEPTING;
 577         return STATUS_SUCCESS;
 578     }
 579
 580     if (!(buffer = gst_buffer_new_wrapped_full(GST_MEMORY_FLAG_READONLY, wg_sample_data(sample), sample->max_size,
 581             0, sample->size, sample, wg_sample_free_notify)))
 582     {
 583         GST_ERROR("Failed to allocate input buffer");
 584         return STATUS_NO_MEMORY;
 585     }
 586     else
 587     {
 588         InterlockedIncrement(&sample->refcount);
 589         GST_INFO("Wrapped %u/%u bytes from sample %p to buffer %p", sample->size, sample->max_size, sample, buffer);
 590     }
 591
 592     if (sample->flags & WG_SAMPLE_FLAG_HAS_PTS)
 593         GST_BUFFER_PTS(buffer) = sample->pts * 100;
 594     if (sample->flags & WG_SAMPLE_FLAG_HAS_DURATION)
 595         GST_BUFFER_DURATION(buffer) = sample->duration * 100;
 596     if (!(sample->flags & WG_SAMPLE_FLAG_SYNC_POINT))
 597         GST_BUFFER_FLAG_SET(buffer, GST_BUFFER_FLAG_DELTA_UNIT);
 598     if (sample->flags & WG_SAMPLE_FLAG_DISCONTINUITY)
 599         GST_BUFFER_FLAG_SET(buffer, GST_BUFFER_FLAG_DISCONT);
 600     gst_atomic_queue_push(transform->input_queue, buffer);
 601
 602     params->result = S_OK;
 603     return STATUS_SUCCESS;
 604 }
 605
 606 static NTSTATUS copy_video_buffer(GstBuffer *buffer, GstCaps *caps, gsize plane_align,
 607         struct wg_sample *sample, gsize *total_size)
 608 {
 609     NTSTATUS status = STATUS_UNSUCCESSFUL;
 610     GstVideoFrame src_frame, dst_frame;
 611     GstVideoInfo src_info, dst_info;
 612     GstVideoAlignment align;
 613     GstBuffer *dst_buffer;
 614
 615     if (!gst_video_info_from_caps(&src_info, caps))
 616     {
 617         GST_ERROR("Failed to get video info from caps.");
 618         return STATUS_UNSUCCESSFUL;
 619     }
 620
 621     dst_info = src_info;
 622     align_video_info_planes(plane_align, &dst_info, &align);
 623
 624     if (sample->max_size < dst_info.size)
 625     {
 626         GST_ERROR("Output buffer is too small.");
 627         return STATUS_BUFFER_TOO_SMALL;
 628     }
 629
 630     if (!(dst_buffer = gst_buffer_new_wrapped_full(0, wg_sample_data(sample), sample->max_size,
 631             0, sample->max_size, 0, NULL)))
 632     {
 633         GST_ERROR("Failed to wrap wg_sample into GstBuffer");
 634         return STATUS_UNSUCCESSFUL;
 635     }
 636     gst_buffer_set_size(dst_buffer, dst_info.size);
 637     *total_size = sample->size = dst_info.size;
 638
 639     if (!gst_video_frame_map(&src_frame, &src_info, buffer, GST_MAP_READ))
 640         GST_ERROR("Failed to map source frame.");
 641     else
 642     {
 643         if (!gst_video_frame_map(&dst_frame, &dst_info, dst_buffer, GST_MAP_WRITE))
 644             GST_ERROR("Failed to map destination frame.");
 645         else
 646         {
 647             if (gst_video_frame_copy(&dst_frame, &src_frame))
 648                 status = STATUS_SUCCESS;
 649             else
 650                 GST_ERROR("Failed to copy video frame.");
 651             gst_video_frame_unmap(&dst_frame);
 652         }
 653         gst_video_frame_unmap(&src_frame);
 654     }
 655
 656     gst_buffer_unref(dst_buffer);
 657     return status;
 658 }
 659
 660 static NTSTATUS copy_buffer(GstBuffer *buffer, GstCaps *caps, struct wg_sample *sample,
 661         gsize *total_size)
 662 {
 663     GstMapInfo info;
 664
 665     if (!gst_buffer_map(buffer, &info, GST_MAP_READ))
 666         return STATUS_UNSUCCESSFUL;
 667
 668     if (sample->max_size >= info.size)
 669         sample->size = info.size;
 670     else
 671     {
 672         sample->flags |= WG_SAMPLE_FLAG_INCOMPLETE;
 673         sample->size = sample->max_size;
 674     }
 675
 676     memcpy(wg_sample_data(sample), info.data, sample->size);
 677     gst_buffer_unmap(buffer, &info);
 678
 679     if (sample->flags & WG_SAMPLE_FLAG_INCOMPLETE)
 680         gst_buffer_resize(buffer, sample->size, -1);
 681
 682     *total_size = info.size;
 683     return STATUS_SUCCESS;
 684 }
 685
 686 static NTSTATUS read_transform_output_data(GstBuffer *buffer, GstCaps *caps, gsize plane_align,
 687         struct wg_sample *sample)
 688 {
 689     gsize total_size;
 690     bool needs_copy;
 691     NTSTATUS status;
 692     GstMapInfo info;
 693
 694     if (!gst_buffer_map(buffer, &info, GST_MAP_READ))
 695     {
 696         GST_ERROR("Failed to map buffer %p", buffer);
 697         sample->size = 0;
 698         return STATUS_UNSUCCESSFUL;
 699     }
 700     needs_copy = info.data != wg_sample_data(sample);
 701     total_size = sample->size = info.size;
 702     gst_buffer_unmap(buffer, &info);
 703
 704     if (!needs_copy)
 705         status = STATUS_SUCCESS;
 706     else if (stream_type_from_caps(caps) == GST_STREAM_TYPE_VIDEO)
 707         status = copy_video_buffer(buffer, caps, plane_align, sample, &total_size);
 708     else
 709         status = copy_buffer(buffer, caps, sample, &total_size);
 710
 711     if (status)
 712     {
 713         GST_ERROR("Failed to copy buffer %p", buffer);
 714         sample->size = 0;
 715         return status;
 716     }
 717
 718     if (GST_BUFFER_PTS_IS_VALID(buffer))
 719     {
 720         sample->flags |= WG_SAMPLE_FLAG_HAS_PTS;
 721         sample->pts = GST_BUFFER_PTS(buffer) / 100;
 722     }
 723     if (GST_BUFFER_DURATION_IS_VALID(buffer))
 724     {
 725         GstClockTime duration = GST_BUFFER_DURATION(buffer) / 100;
 726
 727         duration = (duration * sample->size) / total_size;
 728         GST_BUFFER_DURATION(buffer) -= duration * 100;
 729         if (GST_BUFFER_PTS_IS_VALID(buffer))
 730             GST_BUFFER_PTS(buffer) += duration * 100;
 731
 732         sample->flags |= WG_SAMPLE_FLAG_HAS_DURATION;
 733         sample->duration = duration;
 734     }
 735     if (!GST_BUFFER_FLAG_IS_SET(buffer, GST_BUFFER_FLAG_DELTA_UNIT))
 736         sample->flags |= WG_SAMPLE_FLAG_SYNC_POINT;
 737     if (GST_BUFFER_FLAG_IS_SET(buffer, GST_BUFFER_FLAG_DISCONT))
 738         sample->flags |= WG_SAMPLE_FLAG_DISCONTINUITY;
 739
 740     if (needs_copy)
 741     {
 742         if (stream_type_from_caps(caps) == GST_STREAM_TYPE_VIDEO)
 743             GST_WARNING("Copied %u bytes, sample %p, flags %#x", sample->size, sample, sample->flags);
 744         else
 745             GST_INFO("Copied %u bytes, sample %p, flags %#x", sample->size, sample, sample->flags);
 746     }
 747     else if (sample->flags & WG_SAMPLE_FLAG_INCOMPLETE)
 748         GST_ERROR("Partial read %u bytes, sample %p, flags %#x", sample->size, sample, sample->flags);
 749     else
 750         GST_INFO("Read %u bytes, sample %p, flags %#x", sample->size, sample, sample->flags);
 751
 752     return STATUS_SUCCESS;
 753 }
 754
 755 static bool get_transform_output(struct wg_transform *transform, struct wg_sample *sample)
 756 {
 757     GstBuffer *input_buffer;
 758     GstFlowReturn ret;
 759
 760     wg_allocator_provide_sample(transform->allocator, sample);
 761
 762     while (!(transform->output_sample = gst_atomic_queue_pop(transform->output_queue))
 763             && (input_buffer = gst_atomic_queue_pop(transform->input_queue)))
 764     {
 765         if ((ret = gst_pad_push(transform->my_src, input_buffer)))
 766             GST_WARNING("Failed to push transform input, error %d", ret);
 767     }
 768
 769     /* Remove the sample so the allocator cannot use it */
 770     wg_allocator_provide_sample(transform->allocator, NULL);
 771
 772     return !!transform->output_sample;
 773 }
 774
 775 NTSTATUS wg_transform_read_data(void *args)
 776 {
 777     struct wg_transform_read_data_params *params = args;
 778     struct wg_transform *transform = get_transform(params->transform);
 779     struct wg_sample *sample = params->sample;
 780     struct wg_format *format = params->format;
 781     GstBuffer *output_buffer;
 782     GstCaps *output_caps;
 783     bool discard_data;
 784     NTSTATUS status;
 785
 786     if (!transform->output_sample && !get_transform_output(transform, sample))
 787     {
 788         sample->size = 0;
 789         params->result = MF_E_TRANSFORM_NEED_MORE_INPUT;
 790         GST_INFO("Cannot read %u bytes, no output available", sample->max_size);
 791         wg_allocator_release_sample(transform->allocator, sample, false);
 792         return STATUS_SUCCESS;
 793     }
 794
 795     output_buffer = gst_sample_get_buffer(transform->output_sample);
 796     output_caps = gst_sample_get_caps(transform->output_sample);
 797
 798     if (GST_MINI_OBJECT_FLAG_IS_SET(transform->output_sample, GST_SAMPLE_FLAG_WG_CAPS_CHANGED))
 799     {
 800         GST_MINI_OBJECT_FLAG_UNSET(transform->output_sample, GST_SAMPLE_FLAG_WG_CAPS_CHANGED);
 801
 802         if (format)
 803         {
 804             gsize plane_align = transform->attrs.output_plane_align;
 805             GstVideoAlignment align;
 806             GstVideoInfo info;
 807
 808             wg_format_from_caps(format, output_caps);
 809
 810             if (format->major_type == WG_MAJOR_TYPE_VIDEO
 811                     && gst_video_info_from_caps(&info, output_caps))
 812             {
 813                 align_video_info_planes(plane_align, &info, &align);
 814
 815                 GST_INFO("Returning video alignment left %u, top %u, right %u, bottom %u.", align.padding_left,
 816                         align.padding_top, align.padding_right, align.padding_bottom);
 817
 818                 format->u.video.padding.left = align.padding_left;
 819                 format->u.video.width += format->u.video.padding.left;
 820                 format->u.video.padding.right = align.padding_right;
 821                 format->u.video.width += format->u.video.padding.right;
 822                 format->u.video.padding.top = align.padding_top;
 823                 format->u.video.height += format->u.video.padding.top;
 824                 format->u.video.padding.bottom = align.padding_bottom;
 825                 format->u.video.height += format->u.video.padding.bottom;
 826             }
 827         }
 828
 829         params->result = MF_E_TRANSFORM_STREAM_CHANGE;
 830         GST_INFO("Format changed detected, returning no output");
 831         wg_allocator_release_sample(transform->allocator, sample, false);
 832         return STATUS_SUCCESS;
 833     }
 834
 835     if ((status = read_transform_output_data(output_buffer, output_caps,
 836                 transform->attrs.output_plane_align, sample)))
 837     {
 838         wg_allocator_release_sample(transform->allocator, sample, false);
 839         return status;
 840     }
 841
 842     if (sample->flags & WG_SAMPLE_FLAG_INCOMPLETE)
 843         discard_data = false;
 844     else
 845     {
 846         /* Taint the buffer memory to make sure it cannot be reused by the buffer pool,
 847          * for the pool to always requests new memory from the allocator, and so we can
 848          * then always provide output sample memory to achieve zero-copy.
 849          *
 850          * However, some decoder keep a reference on the buffer they passed downstream,
 851          * to re-use it later. In this case, it will not be possible to do zero-copy,
 852          * and we should copy the data back to the buffer and leave it unchanged.
 853          *
 854          * Some other plugins make assumptions that the returned buffer will always have
 855          * at least one memory attached, we cannot just remove it and need to replace the
 856          * memory instead.
 857          */
 858         if ((discard_data = gst_buffer_is_writable(output_buffer)))
 859             gst_buffer_replace_all_memory(output_buffer, gst_allocator_alloc(NULL, 0, NULL));
 860
 861         gst_sample_unref(transform->output_sample);
 862         transform->output_sample = NULL;
 863     }
 864
 865     params->result = S_OK;
 866     wg_allocator_release_sample(transform->allocator, sample, discard_data);
 867     return STATUS_SUCCESS;
 868 }
 869
 870 NTSTATUS wg_transform_get_status(void *args)
 871 {
 872     struct wg_transform_get_status_params *params = args;
 873     struct wg_transform *transform = get_transform(params->transform);
 874
 875     params->accepts_input = gst_atomic_queue_length(transform->input_queue) < transform->attrs.input_queue_length + 1;
 876     return STATUS_SUCCESS;
 877 }
 878
 879 NTSTATUS wg_transform_drain(void *args)
 880 {
 881     struct wg_transform *transform = get_transform(*(wg_transform_t *)args);
 882     GstBuffer *input_buffer;
 883     GstFlowReturn ret;
 884     GstEvent *event;
 885
 886     GST_LOG("transform %p", transform);
 887
 888     while ((input_buffer = gst_atomic_queue_pop(transform->input_queue)))
 889     {
 890         if ((ret = gst_pad_push(transform->my_src, input_buffer)))
 891             GST_WARNING("Failed to push transform input, error %d", ret);
 892     }
 893
 894     if (!(event = gst_event_new_segment_done(GST_FORMAT_TIME, -1))
 895             || !gst_pad_push_event(transform->my_src, event))
 896         goto error;
 897     if (!(event = gst_event_new_eos())
 898             || !gst_pad_push_event(transform->my_src, event))
 899         goto error;
 900     if (!(event = gst_event_new_stream_start("stream"))
 901             || !gst_pad_push_event(transform->my_src, event))
 902         goto error;
 903     if (!(event = gst_event_new_segment(&transform->segment))
 904             || !gst_pad_push_event(transform->my_src, event))
 905         goto error;
 906
 907     return STATUS_SUCCESS;
 908
 909 error:
 910     GST_ERROR("Failed to drain transform %p.", transform);
 911     return STATUS_UNSUCCESSFUL;
 912 }
 913
 914 NTSTATUS wg_transform_flush(void *args)
 915 {
 916     struct wg_transform *transform = get_transform(*(wg_transform_t *)args);
 917     GstBuffer *input_buffer;
 918     GstSample *sample;
 919     NTSTATUS status;
 920
 921     GST_LOG("transform %p", transform);
 922
 923     while ((input_buffer = gst_atomic_queue_pop(transform->input_queue)))
 924         gst_buffer_unref(input_buffer);
 925
 926     if ((status = wg_transform_drain(args)))
 927         return status;
 928
 929     while ((sample = gst_atomic_queue_pop(transform->output_queue)))
 930         gst_sample_unref(sample);
 931     if ((sample = transform->output_sample))
 932         gst_sample_unref(sample);
 933     transform->output_sample = NULL;
 934
 935     return STATUS_SUCCESS;
 936 }