dlls/winegstreamer/wg_transform.c

   1 /*
   2  * GStreamer transform backend
   3  *
   4  * Copyright 2022 Rémi Bernon for CodeWeavers
   5  *
   6  * This library is free software; you can redistribute it and/or
   7  * modify it under the terms of the GNU Lesser General Public
   8  * License as published by the Free Software Foundation; either
   9  * version 2.1 of the License, or (at your option) any later version.
  10  *
  11  * This library is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  14  * Lesser General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU Lesser General Public
  17  * License along with this library; if not, write to the Free Software
  18  * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301, USA
  19  */
  20
  21 #if 0
  22 #pragma makedep unix
  23 #endif
  24
  25 #include "config.h"
  26
  27 #include <assert.h>
  28 #include <stdarg.h>
  29 #include <stdio.h>
  30
  31 #include <gst/gst.h>
  32 #include <gst/video/video.h>
  33 #include <gst/audio/audio.h>
  34
  35 #include "ntstatus.h"
  36 #define WIN32_NO_STATUS
  37 #include "winternl.h"
  38 #include "mferror.h"
  39
  40 #include "unix_private.h"
  41
  42 #define GST_SAMPLE_FLAG_WG_CAPS_CHANGED (GST_MINI_OBJECT_FLAG_LAST << 0)
  43
  44 struct wg_transform
  45 {
  46     struct wg_transform_attrs attrs;
  47
  48     GstElement *container;
  49     GstAllocator *allocator;
  50     GstPad *my_src, *my_sink;
  51     GstSegment segment;
  52     GstQuery *drain_query;
  53
  54     GstAtomicQueue *input_queue;
  55
  56     bool input_is_flipped;
  57     GstElement *video_flip;
  58
  59     struct wg_format output_format;
  60     GstAtomicQueue *output_queue;
  61     GstSample *output_sample;
  62     bool output_caps_changed;
  63     GstCaps *output_caps;
  64 };
  65
  66 static struct wg_transform *get_transform(wg_transform_t trans)
  67 {
  68     return (struct wg_transform *)(ULONG_PTR)trans;
  69 }
  70
  71 static void align_video_info_planes(gsize plane_align, GstVideoInfo *info, GstVideoAlignment *align)
  72 {
  73     gst_video_alignment_reset(align);
  74
  75     align->padding_right = ((plane_align + 1) - (info->width & plane_align)) & plane_align;
  76     align->padding_bottom = ((plane_align + 1) - (info->height & plane_align)) & plane_align;
  77     align->stride_align[0] = plane_align;
  78     align->stride_align[1] = plane_align;
  79     align->stride_align[2] = plane_align;
  80     align->stride_align[3] = plane_align;
  81
  82     gst_video_info_align(info, align);
  83 }
  84
  85 static GstFlowReturn transform_sink_chain_cb(GstPad *pad, GstObject *parent, GstBuffer *buffer)
  86 {
  87     struct wg_transform *transform = gst_pad_get_element_private(pad);
  88     GstSample *sample;
  89
  90     GST_LOG("transform %p, buffer %p.", transform, buffer);
  91
  92     if (!(sample = gst_sample_new(buffer, transform->output_caps, NULL, NULL)))
  93     {
  94         GST_ERROR("Failed to allocate transform %p output sample.", transform);
  95         gst_buffer_unref(buffer);
  96         return GST_FLOW_ERROR;
  97     }
  98
  99     if (transform->output_caps_changed)
 100         GST_MINI_OBJECT_FLAG_SET(sample, GST_SAMPLE_FLAG_WG_CAPS_CHANGED);
 101     transform->output_caps_changed = false;
 102
 103     gst_atomic_queue_push(transform->output_queue, sample);
 104     gst_buffer_unref(buffer);
 105     return GST_FLOW_OK;
 106 }
 107
 108 static gboolean transform_src_query_latency(struct wg_transform *transform, GstQuery *query)
 109 {
 110     GST_LOG("transform %p, query %p", transform, query);
 111     gst_query_set_latency(query, transform->attrs.low_latency, 0, 0);
 112     return true;
 113 }
 114
 115 static gboolean transform_src_query_cb(GstPad *pad, GstObject *parent, GstQuery *query)
 116 {
 117     struct wg_transform *transform = gst_pad_get_element_private(pad);
 118
 119     switch (query->type)
 120     {
 121     case GST_QUERY_LATENCY:
 122         return transform_src_query_latency(transform, query);
 123     default:
 124         return gst_pad_query_default(pad, parent, query);
 125     }
 126 }
 127
 128 static gboolean transform_sink_query_cb(GstPad *pad, GstObject *parent, GstQuery *query)
 129 {
 130     struct wg_transform *transform = gst_pad_get_element_private(pad);
 131
 132     GST_LOG("transform %p, type \"%s\".", transform, gst_query_type_get_name(query->type));
 133
 134     switch (query->type)
 135     {
 136         case GST_QUERY_ALLOCATION:
 137         {
 138             gsize plane_align = transform->attrs.output_plane_align;
 139             GstStructure *config, *params;
 140             GstVideoAlignment align;
 141             gboolean needs_pool;
 142             GstBufferPool *pool;
 143             GstVideoInfo info;
 144             GstCaps *caps;
 145
 146             gst_query_parse_allocation(query, &caps, &needs_pool);
 147             if (stream_type_from_caps(caps) != GST_STREAM_TYPE_VIDEO || !needs_pool)
 148                 break;
 149
 150             if (!gst_video_info_from_caps(&info, caps)
 151                     || !(pool = gst_video_buffer_pool_new()))
 152                 break;
 153
 154             align_video_info_planes(plane_align, &info, &align);
 155
 156             if ((params = gst_structure_new("video-meta",
 157                     "padding-top", G_TYPE_UINT, align.padding_top,
 158                     "padding-bottom", G_TYPE_UINT, align.padding_bottom,
 159                     "padding-left", G_TYPE_UINT, align.padding_left,
 160                     "padding-right", G_TYPE_UINT, align.padding_right,
 161                     NULL)))
 162             {
 163                 gst_query_add_allocation_meta(query, GST_VIDEO_META_API_TYPE, params);
 164                 gst_structure_free(params);
 165             }
 166
 167             if (!(config = gst_buffer_pool_get_config(pool)))
 168                 GST_ERROR("Failed to get pool %p config.", pool);
 169             else
 170             {
 171                 gst_buffer_pool_config_add_option(config, GST_BUFFER_POOL_OPTION_VIDEO_META);
 172                 gst_buffer_pool_config_add_option(config, GST_BUFFER_POOL_OPTION_VIDEO_ALIGNMENT);
 173                 gst_buffer_pool_config_set_video_alignment(config, &align);
 174
 175                 gst_buffer_pool_config_set_params(config, caps,
 176                         info.size, 0, 0);
 177                 gst_buffer_pool_config_set_allocator(config, transform->allocator, NULL);
 178                 if (!gst_buffer_pool_set_config(pool, config))
 179                     GST_ERROR("Failed to set pool %p config.", pool);
 180             }
 181
 182             /* Prevent pool reconfiguration, we don't want another alignment. */
 183             if (!gst_buffer_pool_set_active(pool, true))
 184                 GST_ERROR("Pool %p failed to activate.", pool);
 185
 186             gst_query_add_allocation_pool(query, pool, info.size, 0, 0);
 187             gst_query_add_allocation_param(query, transform->allocator, NULL);
 188
 189             GST_INFO("Proposing pool %p, buffer size %#zx, allocator %p, for query %p.",
 190                     pool, info.size, transform->allocator, query);
 191
 192             g_object_unref(pool);
 193             return true;
 194         }
 195
 196         case GST_QUERY_CAPS:
 197         {
 198             GstCaps *caps, *filter, *temp;
 199
 200             gst_query_parse_caps(query, &filter);
 201             if (!(caps = wg_format_to_caps(&transform->output_format)))
 202                 break;
 203
 204             if (filter)
 205             {
 206                 temp = gst_caps_intersect(caps, filter);
 207                 gst_caps_unref(caps);
 208                 caps = temp;
 209             }
 210
 211             GST_INFO("Returning caps %" GST_PTR_FORMAT, caps);
 212
 213             gst_query_set_caps_result(query, caps);
 214             gst_caps_unref(caps);
 215             return true;
 216         }
 217
 218         default:
 219             GST_WARNING("Ignoring \"%s\" query.", gst_query_type_get_name(query->type));
 220             break;
 221     }
 222
 223     return gst_pad_query_default(pad, parent, query);
 224 }
 225
 226 static gboolean transform_sink_event_cb(GstPad *pad, GstObject *parent, GstEvent *event)
 227 {
 228     struct wg_transform *transform = gst_pad_get_element_private(pad);
 229
 230     GST_LOG("transform %p, type \"%s\".", transform, GST_EVENT_TYPE_NAME(event));
 231
 232     switch (event->type)
 233     {
 234         case GST_EVENT_CAPS:
 235         {
 236             GstCaps *caps;
 237
 238             gst_event_parse_caps(event, &caps);
 239
 240             transform->output_caps_changed = transform->output_caps_changed
 241                     || !gst_caps_is_always_compatible(transform->output_caps, caps);
 242
 243             gst_caps_unref(transform->output_caps);
 244             transform->output_caps = gst_caps_ref(caps);
 245             break;
 246         }
 247         default:
 248             GST_WARNING("Ignoring \"%s\" event.", GST_EVENT_TYPE_NAME(event));
 249             break;
 250     }
 251
 252     gst_event_unref(event);
 253     return TRUE;
 254 }
 255
 256 NTSTATUS wg_transform_destroy(void *args)
 257 {
 258     struct wg_transform *transform = get_transform(*(wg_transform_t *)args);
 259     GstSample *sample;
 260     GstBuffer *buffer;
 261
 262     while ((buffer = gst_atomic_queue_pop(transform->input_queue)))
 263         gst_buffer_unref(buffer);
 264     gst_atomic_queue_unref(transform->input_queue);
 265
 266     gst_element_set_state(transform->container, GST_STATE_NULL);
 267
 268     if (transform->output_sample)
 269         gst_sample_unref(transform->output_sample);
 270     while ((sample = gst_atomic_queue_pop(transform->output_queue)))
 271         gst_sample_unref(sample);
 272
 273     wg_allocator_destroy(transform->allocator);
 274     g_object_unref(transform->container);
 275     g_object_unref(transform->my_sink);
 276     g_object_unref(transform->my_src);
 277     gst_query_unref(transform->drain_query);
 278     gst_caps_unref(transform->output_caps);
 279     gst_atomic_queue_unref(transform->output_queue);
 280     free(transform);
 281
 282     return STATUS_SUCCESS;
 283 }
 284
 285 static bool wg_format_video_is_flipped(const struct wg_format *format)
 286 {
 287     return format->major_type == WG_MAJOR_TYPE_VIDEO && (format->u.video.height < 0);
 288 }
 289
 290 NTSTATUS wg_transform_create(void *args)
 291 {
 292     struct wg_transform_create_params *params = args;
 293     struct wg_format output_format = *params->output_format;
 294     struct wg_format input_format = *params->input_format;
 295     GstElement *first = NULL, *last = NULL, *element;
 296     GstCaps *raw_caps = NULL, *src_caps = NULL;
 297     NTSTATUS status = STATUS_UNSUCCESSFUL;
 298     GstPadTemplate *template = NULL;
 299     struct wg_transform *transform;
 300     const gchar *media_type;
 301     GstEvent *event;
 302
 303     if (!(transform = calloc(1, sizeof(*transform))))
 304         return STATUS_NO_MEMORY;
 305     if (!(transform->container = gst_bin_new("wg_transform")))
 306         goto out;
 307     if (!(transform->input_queue = gst_atomic_queue_new(8)))
 308         goto out;
 309     if (!(transform->output_queue = gst_atomic_queue_new(8)))
 310         goto out;
 311     if (!(transform->drain_query = gst_query_new_drain()))
 312         goto out;
 313     if (!(transform->allocator = wg_allocator_create()))
 314         goto out;
 315     transform->attrs = *params->attrs;
 316     transform->output_format = output_format;
 317
 318     if (!(src_caps = wg_format_to_caps(&input_format)))
 319         goto out;
 320     if (!(template = gst_pad_template_new("src", GST_PAD_SRC, GST_PAD_ALWAYS, src_caps)))
 321         goto out;
 322     transform->my_src = gst_pad_new_from_template(template, "src");
 323     g_object_unref(template);
 324     if (!transform->my_src)
 325         goto out;
 326
 327     gst_pad_set_element_private(transform->my_src, transform);
 328     gst_pad_set_query_function(transform->my_src, transform_src_query_cb);
 329
 330     if (!(transform->output_caps = wg_format_to_caps(&output_format)))
 331         goto out;
 332     if (!(template = gst_pad_template_new("sink", GST_PAD_SINK, GST_PAD_ALWAYS, transform->output_caps)))
 333         goto out;
 334     transform->my_sink = gst_pad_new_from_template(template, "sink");
 335     g_object_unref(template);
 336     if (!transform->my_sink)
 337         goto out;
 338
 339     gst_pad_set_element_private(transform->my_sink, transform);
 340     gst_pad_set_event_function(transform->my_sink, transform_sink_event_cb);
 341     gst_pad_set_query_function(transform->my_sink, transform_sink_query_cb);
 342     gst_pad_set_chain_function(transform->my_sink, transform_sink_chain_cb);
 343
 344     /* Since we append conversion elements, we don't want to filter decoders
 345      * based on the actual output caps now. Matching decoders with the
 346      * raw output media type should be enough.
 347      */
 348     media_type = gst_structure_get_name(gst_caps_get_structure(transform->output_caps, 0));
 349     if (!(raw_caps = gst_caps_new_empty_simple(media_type)))
 350         goto out;
 351
 352     switch (input_format.major_type)
 353     {
 354         case WG_MAJOR_TYPE_VIDEO_H264:
 355         case WG_MAJOR_TYPE_AUDIO_MPEG1:
 356         case WG_MAJOR_TYPE_AUDIO_MPEG4:
 357         case WG_MAJOR_TYPE_AUDIO_WMA:
 358         case WG_MAJOR_TYPE_VIDEO_CINEPAK:
 359         case WG_MAJOR_TYPE_VIDEO_INDEO:
 360         case WG_MAJOR_TYPE_VIDEO_WMV:
 361         case WG_MAJOR_TYPE_VIDEO_MPEG1:
 362             if (!(element = find_element(GST_ELEMENT_FACTORY_TYPE_DECODER, src_caps, raw_caps))
 363                     || !append_element(transform->container, element, &first, &last))
 364             {
 365                 gst_caps_unref(raw_caps);
 366                 goto out;
 367             }
 368             break;
 369
 370         case WG_MAJOR_TYPE_AUDIO:
 371         case WG_MAJOR_TYPE_VIDEO:
 372             break;
 373         case WG_MAJOR_TYPE_UNKNOWN:
 374             GST_FIXME("Format %u not implemented!", input_format.major_type);
 375             gst_caps_unref(raw_caps);
 376             goto out;
 377     }
 378
 379     gst_caps_unref(raw_caps);
 380
 381     switch (output_format.major_type)
 382     {
 383         case WG_MAJOR_TYPE_AUDIO:
 384             /* The MF audio decoder transforms allow decoding to various formats
 385              * as well as resampling the audio at the same time, whereas
 386              * GStreamer decoder plugins usually only support decoding to a
 387              * single format and at the original rate.
 388              *
 389              * The WMA decoder transform also has output samples interleaved on
 390              * Windows, whereas GStreamer avdec_wmav2 output uses
 391              * non-interleaved format.
 392              */
 393             if (!(element = create_element("audioconvert", "base"))
 394                     || !append_element(transform->container, element, &first, &last))
 395                 goto out;
 396             if (!(element = create_element("audioresample", "base"))
 397                     || !append_element(transform->container, element, &first, &last))
 398                 goto out;
 399             break;
 400
 401         case WG_MAJOR_TYPE_VIDEO:
 402             if (!(element = create_element("videoconvert", "base"))
 403                     || !append_element(transform->container, element, &first, &last))
 404                 goto out;
 405             if (!(transform->video_flip = create_element("videoflip", "base"))
 406                     || !append_element(transform->container, transform->video_flip, &first, &last))
 407                 goto out;
 408             transform->input_is_flipped = wg_format_video_is_flipped(&input_format);
 409             if (transform->input_is_flipped != wg_format_video_is_flipped(&output_format))
 410                 gst_util_set_object_arg(G_OBJECT(transform->video_flip), "method", "vertical-flip");
 411             if (!(element = create_element("videoconvert", "base"))
 412                     || !append_element(transform->container, element, &first, &last))
 413                 goto out;
 414             /* Let GStreamer choose a default number of threads. */
 415             gst_util_set_object_arg(G_OBJECT(element), "n-threads", "0");
 416             break;
 417
 418         case WG_MAJOR_TYPE_UNKNOWN:
 419         case WG_MAJOR_TYPE_AUDIO_MPEG1:
 420         case WG_MAJOR_TYPE_AUDIO_MPEG4:
 421         case WG_MAJOR_TYPE_AUDIO_WMA:
 422         case WG_MAJOR_TYPE_VIDEO_CINEPAK:
 423         case WG_MAJOR_TYPE_VIDEO_H264:
 424         case WG_MAJOR_TYPE_VIDEO_INDEO:
 425         case WG_MAJOR_TYPE_VIDEO_WMV:
 426         case WG_MAJOR_TYPE_VIDEO_MPEG1:
 427             GST_FIXME("Format %u not implemented!", output_format.major_type);
 428             goto out;
 429     }
 430
 431     if (!link_src_to_element(transform->my_src, first))
 432         goto out;
 433     if (!link_element_to_sink(last, transform->my_sink))
 434         goto out;
 435     if (!gst_pad_set_active(transform->my_sink, 1))
 436         goto out;
 437     if (!gst_pad_set_active(transform->my_src, 1))
 438         goto out;
 439
 440     gst_element_set_state(transform->container, GST_STATE_PAUSED);
 441     if (!gst_element_get_state(transform->container, NULL, NULL, -1))
 442         goto out;
 443
 444     if (!(event = gst_event_new_stream_start("stream"))
 445             || !push_event(transform->my_src, event))
 446         goto out;
 447     if (!(event = gst_event_new_caps(src_caps))
 448             || !push_event(transform->my_src, event))
 449         goto out;
 450
 451     /* We need to use GST_FORMAT_TIME here because it's the only format
 452      * some elements such avdec_wmav2 correctly support. */
 453     gst_segment_init(&transform->segment, GST_FORMAT_TIME);
 454     transform->segment.start = 0;
 455     transform->segment.stop = -1;
 456     if (!(event = gst_event_new_segment(&transform->segment))
 457             || !push_event(transform->my_src, event))
 458         goto out;
 459
 460     gst_caps_unref(src_caps);
 461
 462     GST_INFO("Created winegstreamer transform %p.", transform);
 463     params->transform = (wg_transform_t)(ULONG_PTR)transform;
 464     return STATUS_SUCCESS;
 465
 466 out:
 467     if (transform->my_sink)
 468         gst_object_unref(transform->my_sink);
 469     if (transform->output_caps)
 470         gst_caps_unref(transform->output_caps);
 471     if (transform->my_src)
 472         gst_object_unref(transform->my_src);
 473     if (src_caps)
 474         gst_caps_unref(src_caps);
 475     if (transform->allocator)
 476         wg_allocator_destroy(transform->allocator);
 477     if (transform->drain_query)
 478         gst_query_unref(transform->drain_query);
 479     if (transform->output_queue)
 480         gst_atomic_queue_unref(transform->output_queue);
 481     if (transform->input_queue)
 482         gst_atomic_queue_unref(transform->input_queue);
 483     if (transform->container)
 484     {
 485         gst_element_set_state(transform->container, GST_STATE_NULL);
 486         gst_object_unref(transform->container);
 487     }
 488     free(transform);
 489     GST_ERROR("Failed to create winegstreamer transform.");
 490     return status;
 491 }
 492
 493 NTSTATUS wg_transform_set_output_format(void *args)
 494 {
 495     struct wg_transform_set_output_format_params *params = args;
 496     struct wg_transform *transform = get_transform(params->transform);
 497     const struct wg_format *format = params->format;
 498     GstSample *sample;
 499     GstCaps *caps;
 500
 501     if (!(caps = wg_format_to_caps(format)))
 502     {
 503         GST_ERROR("Failed to convert format %p to caps.", format);
 504         return STATUS_UNSUCCESSFUL;
 505     }
 506     transform->output_format = *format;
 507
 508     if (gst_caps_is_always_compatible(transform->output_caps, caps))
 509     {
 510         gst_caps_unref(caps);
 511         return STATUS_SUCCESS;
 512     }
 513
 514     if (!gst_pad_peer_query(transform->my_src, transform->drain_query))
 515     {
 516         GST_ERROR("Failed to drain transform %p.", transform);
 517         return STATUS_UNSUCCESSFUL;
 518     }
 519
 520     gst_caps_unref(transform->output_caps);
 521     transform->output_caps = caps;
 522
 523     if (transform->video_flip)
 524     {
 525         const char *value;
 526         if (transform->input_is_flipped != wg_format_video_is_flipped(format))
 527             value = "vertical-flip";
 528         else
 529             value = "none";
 530         gst_util_set_object_arg(G_OBJECT(transform->video_flip), "method", value);
 531     }
 532     if (!push_event(transform->my_sink, gst_event_new_reconfigure()))
 533     {
 534         GST_ERROR("Failed to reconfigure transform %p.", transform);
 535         return STATUS_UNSUCCESSFUL;
 536     }
 537
 538     GST_INFO("Configured new caps %" GST_PTR_FORMAT ".", caps);
 539
 540     /* Ideally and to be fully compatible with native transform, the queued
 541      * output buffers will need to be converted to the new output format and
 542      * kept queued.
 543      */
 544     if (transform->output_sample)
 545         gst_sample_unref(transform->output_sample);
 546     while ((sample = gst_atomic_queue_pop(transform->output_queue)))
 547         gst_sample_unref(sample);
 548     transform->output_sample = NULL;
 549
 550     return STATUS_SUCCESS;
 551 }
 552
 553 static void wg_sample_free_notify(void *arg)
 554 {
 555     struct wg_sample *sample = arg;
 556     GST_DEBUG("Releasing wg_sample %p", sample);
 557     InterlockedDecrement(&sample->refcount);
 558 }
 559
 560 NTSTATUS wg_transform_push_data(void *args)
 561 {
 562     struct wg_transform_push_data_params *params = args;
 563     struct wg_transform *transform = get_transform(params->transform);
 564     struct wg_sample *sample = params->sample;
 565     GstBuffer *buffer;
 566     guint length;
 567
 568     length = gst_atomic_queue_length(transform->input_queue);
 569     if (length >= transform->attrs.input_queue_length + 1)
 570     {
 571         GST_INFO("Refusing %u bytes, %u buffers already queued", sample->size, length);
 572         params->result = MF_E_NOTACCEPTING;
 573         return STATUS_SUCCESS;
 574     }
 575
 576     if (!(buffer = gst_buffer_new_wrapped_full(GST_MEMORY_FLAG_READONLY, wg_sample_data(sample), sample->max_size,
 577             0, sample->size, sample, wg_sample_free_notify)))
 578     {
 579         GST_ERROR("Failed to allocate input buffer");
 580         return STATUS_NO_MEMORY;
 581     }
 582     else
 583     {
 584         InterlockedIncrement(&sample->refcount);
 585         GST_INFO("Wrapped %u/%u bytes from sample %p to buffer %p", sample->size, sample->max_size, sample, buffer);
 586     }
 587
 588     if (sample->flags & WG_SAMPLE_FLAG_HAS_PTS)
 589         GST_BUFFER_PTS(buffer) = sample->pts * 100;
 590     if (sample->flags & WG_SAMPLE_FLAG_HAS_DURATION)
 591         GST_BUFFER_DURATION(buffer) = sample->duration * 100;
 592     if (!(sample->flags & WG_SAMPLE_FLAG_SYNC_POINT))
 593         GST_BUFFER_FLAG_SET(buffer, GST_BUFFER_FLAG_DELTA_UNIT);
 594     if (sample->flags & WG_SAMPLE_FLAG_DISCONTINUITY)
 595         GST_BUFFER_FLAG_SET(buffer, GST_BUFFER_FLAG_DISCONT);
 596     gst_atomic_queue_push(transform->input_queue, buffer);
 597
 598     params->result = S_OK;
 599     return STATUS_SUCCESS;
 600 }
 601
 602 static NTSTATUS copy_video_buffer(GstBuffer *buffer, GstCaps *caps, gsize plane_align,
 603         struct wg_sample *sample, gsize *total_size)
 604 {
 605     NTSTATUS status = STATUS_UNSUCCESSFUL;
 606     GstVideoFrame src_frame, dst_frame;
 607     GstVideoInfo src_info, dst_info;
 608     GstVideoAlignment align;
 609     GstBuffer *dst_buffer;
 610
 611     if (!gst_video_info_from_caps(&src_info, caps))
 612     {
 613         GST_ERROR("Failed to get video info from caps.");
 614         return STATUS_UNSUCCESSFUL;
 615     }
 616
 617     dst_info = src_info;
 618     align_video_info_planes(plane_align, &dst_info, &align);
 619
 620     if (sample->max_size < dst_info.size)
 621     {
 622         GST_ERROR("Output buffer is too small.");
 623         return STATUS_BUFFER_TOO_SMALL;
 624     }
 625
 626     if (!(dst_buffer = gst_buffer_new_wrapped_full(0, wg_sample_data(sample), sample->max_size,
 627             0, sample->max_size, 0, NULL)))
 628     {
 629         GST_ERROR("Failed to wrap wg_sample into GstBuffer");
 630         return STATUS_UNSUCCESSFUL;
 631     }
 632     gst_buffer_set_size(dst_buffer, dst_info.size);
 633     *total_size = sample->size = dst_info.size;
 634
 635     if (!gst_video_frame_map(&src_frame, &src_info, buffer, GST_MAP_READ))
 636         GST_ERROR("Failed to map source frame.");
 637     else
 638     {
 639         if (!gst_video_frame_map(&dst_frame, &dst_info, dst_buffer, GST_MAP_WRITE))
 640             GST_ERROR("Failed to map destination frame.");
 641         else
 642         {
 643             if (gst_video_frame_copy(&dst_frame, &src_frame))
 644                 status = STATUS_SUCCESS;
 645             else
 646                 GST_ERROR("Failed to copy video frame.");
 647             gst_video_frame_unmap(&dst_frame);
 648         }
 649         gst_video_frame_unmap(&src_frame);
 650     }
 651
 652     gst_buffer_unref(dst_buffer);
 653     return status;
 654 }
 655
 656 static NTSTATUS copy_buffer(GstBuffer *buffer, GstCaps *caps, struct wg_sample *sample,
 657         gsize *total_size)
 658 {
 659     GstMapInfo info;
 660
 661     if (!gst_buffer_map(buffer, &info, GST_MAP_READ))
 662         return STATUS_UNSUCCESSFUL;
 663
 664     if (sample->max_size >= info.size)
 665         sample->size = info.size;
 666     else
 667     {
 668         sample->flags |= WG_SAMPLE_FLAG_INCOMPLETE;
 669         sample->size = sample->max_size;
 670     }
 671
 672     memcpy(wg_sample_data(sample), info.data, sample->size);
 673     gst_buffer_unmap(buffer, &info);
 674
 675     if (sample->flags & WG_SAMPLE_FLAG_INCOMPLETE)
 676         gst_buffer_resize(buffer, sample->size, -1);
 677
 678     *total_size = info.size;
 679     return STATUS_SUCCESS;
 680 }
 681
 682 static NTSTATUS read_transform_output_data(GstBuffer *buffer, GstCaps *caps, gsize plane_align,
 683         struct wg_sample *sample)
 684 {
 685     gsize total_size;
 686     bool needs_copy;
 687     NTSTATUS status;
 688     GstMapInfo info;
 689
 690     if (!gst_buffer_map(buffer, &info, GST_MAP_READ))
 691     {
 692         GST_ERROR("Failed to map buffer %p", buffer);
 693         sample->size = 0;
 694         return STATUS_UNSUCCESSFUL;
 695     }
 696     needs_copy = info.data != wg_sample_data(sample);
 697     total_size = sample->size = info.size;
 698     gst_buffer_unmap(buffer, &info);
 699
 700     if (!needs_copy)
 701         status = STATUS_SUCCESS;
 702     else if (stream_type_from_caps(caps) == GST_STREAM_TYPE_VIDEO)
 703         status = copy_video_buffer(buffer, caps, plane_align, sample, &total_size);
 704     else
 705         status = copy_buffer(buffer, caps, sample, &total_size);
 706
 707     if (status)
 708     {
 709         GST_ERROR("Failed to copy buffer %p", buffer);
 710         sample->size = 0;
 711         return status;
 712     }
 713
 714     if (GST_BUFFER_PTS_IS_VALID(buffer))
 715     {
 716         sample->flags |= WG_SAMPLE_FLAG_HAS_PTS;
 717         sample->pts = GST_BUFFER_PTS(buffer) / 100;
 718     }
 719     if (GST_BUFFER_DURATION_IS_VALID(buffer))
 720     {
 721         GstClockTime duration = GST_BUFFER_DURATION(buffer) / 100;
 722
 723         duration = (duration * sample->size) / total_size;
 724         GST_BUFFER_DURATION(buffer) -= duration * 100;
 725         if (GST_BUFFER_PTS_IS_VALID(buffer))
 726             GST_BUFFER_PTS(buffer) += duration * 100;
 727
 728         sample->flags |= WG_SAMPLE_FLAG_HAS_DURATION;
 729         sample->duration = duration;
 730     }
 731     if (!GST_BUFFER_FLAG_IS_SET(buffer, GST_BUFFER_FLAG_DELTA_UNIT))
 732         sample->flags |= WG_SAMPLE_FLAG_SYNC_POINT;
 733     if (GST_BUFFER_FLAG_IS_SET(buffer, GST_BUFFER_FLAG_DISCONT))
 734         sample->flags |= WG_SAMPLE_FLAG_DISCONTINUITY;
 735
 736     if (needs_copy)
 737     {
 738         if (stream_type_from_caps(caps) == GST_STREAM_TYPE_VIDEO)
 739             GST_WARNING("Copied %u bytes, sample %p, flags %#x", sample->size, sample, sample->flags);
 740         else
 741             GST_INFO("Copied %u bytes, sample %p, flags %#x", sample->size, sample, sample->flags);
 742     }
 743     else if (sample->flags & WG_SAMPLE_FLAG_INCOMPLETE)
 744         GST_ERROR("Partial read %u bytes, sample %p, flags %#x", sample->size, sample, sample->flags);
 745     else
 746         GST_INFO("Read %u bytes, sample %p, flags %#x", sample->size, sample, sample->flags);
 747
 748     return STATUS_SUCCESS;
 749 }
 750
 751 static bool get_transform_output(struct wg_transform *transform, struct wg_sample *sample)
 752 {
 753     GstBuffer *input_buffer;
 754     GstFlowReturn ret;
 755
 756     wg_allocator_provide_sample(transform->allocator, sample);
 757
 758     while (!(transform->output_sample = gst_atomic_queue_pop(transform->output_queue))
 759             && (input_buffer = gst_atomic_queue_pop(transform->input_queue)))
 760     {
 761         if ((ret = gst_pad_push(transform->my_src, input_buffer)))
 762             GST_WARNING("Failed to push transform input, error %d", ret);
 763     }
 764
 765     /* Remove the sample so the allocator cannot use it */
 766     wg_allocator_provide_sample(transform->allocator, NULL);
 767
 768     return !!transform->output_sample;
 769 }
 770
 771 NTSTATUS wg_transform_read_data(void *args)
 772 {
 773     struct wg_transform_read_data_params *params = args;
 774     struct wg_transform *transform = get_transform(params->transform);
 775     struct wg_sample *sample = params->sample;
 776     struct wg_format *format = params->format;
 777     GstBuffer *output_buffer;
 778     GstCaps *output_caps;
 779     bool discard_data;
 780     NTSTATUS status;
 781
 782     if (!transform->output_sample && !get_transform_output(transform, sample))
 783     {
 784         sample->size = 0;
 785         params->result = MF_E_TRANSFORM_NEED_MORE_INPUT;
 786         GST_INFO("Cannot read %u bytes, no output available", sample->max_size);
 787         wg_allocator_release_sample(transform->allocator, sample, false);
 788         return STATUS_SUCCESS;
 789     }
 790
 791     output_buffer = gst_sample_get_buffer(transform->output_sample);
 792     output_caps = gst_sample_get_caps(transform->output_sample);
 793
 794     if (GST_MINI_OBJECT_FLAG_IS_SET(transform->output_sample, GST_SAMPLE_FLAG_WG_CAPS_CHANGED))
 795     {
 796         GST_MINI_OBJECT_FLAG_UNSET(transform->output_sample, GST_SAMPLE_FLAG_WG_CAPS_CHANGED);
 797
 798         if (format)
 799         {
 800             gsize plane_align = transform->attrs.output_plane_align;
 801             GstVideoAlignment align;
 802             GstVideoInfo info;
 803
 804             wg_format_from_caps(format, output_caps);
 805
 806             if (format->major_type == WG_MAJOR_TYPE_VIDEO
 807                     && gst_video_info_from_caps(&info, output_caps))
 808             {
 809                 align_video_info_planes(plane_align, &info, &align);
 810
 811                 GST_INFO("Returning video alignment left %u, top %u, right %u, bottom %u.", align.padding_left,
 812                         align.padding_top, align.padding_right, align.padding_bottom);
 813
 814                 format->u.video.padding.left = align.padding_left;
 815                 format->u.video.width += format->u.video.padding.left;
 816                 format->u.video.padding.right = align.padding_right;
 817                 format->u.video.width += format->u.video.padding.right;
 818                 format->u.video.padding.top = align.padding_top;
 819                 format->u.video.height += format->u.video.padding.top;
 820                 format->u.video.padding.bottom = align.padding_bottom;
 821                 format->u.video.height += format->u.video.padding.bottom;
 822             }
 823         }
 824
 825         params->result = MF_E_TRANSFORM_STREAM_CHANGE;
 826         GST_INFO("Format changed detected, returning no output");
 827         wg_allocator_release_sample(transform->allocator, sample, false);
 828         return STATUS_SUCCESS;
 829     }
 830
 831     if ((status = read_transform_output_data(output_buffer, output_caps,
 832                 transform->attrs.output_plane_align, sample)))
 833     {
 834         wg_allocator_release_sample(transform->allocator, sample, false);
 835         return status;
 836     }
 837
 838     if (sample->flags & WG_SAMPLE_FLAG_INCOMPLETE)
 839         discard_data = false;
 840     else
 841     {
 842         /* Taint the buffer memory to make sure it cannot be reused by the buffer pool,
 843          * for the pool to always requests new memory from the allocator, and so we can
 844          * then always provide output sample memory to achieve zero-copy.
 845          *
 846          * However, some decoder keep a reference on the buffer they passed downstream,
 847          * to re-use it later. In this case, it will not be possible to do zero-copy,
 848          * and we should copy the data back to the buffer and leave it unchanged.
 849          *
 850          * Some other plugins make assumptions that the returned buffer will always have
 851          * at least one memory attached, we cannot just remove it and need to replace the
 852          * memory instead.
 853          */
 854         if ((discard_data = gst_buffer_is_writable(output_buffer)))
 855             gst_buffer_replace_all_memory(output_buffer, gst_allocator_alloc(NULL, 0, NULL));
 856
 857         gst_sample_unref(transform->output_sample);
 858         transform->output_sample = NULL;
 859     }
 860
 861     params->result = S_OK;
 862     wg_allocator_release_sample(transform->allocator, sample, discard_data);
 863     return STATUS_SUCCESS;
 864 }
 865
 866 NTSTATUS wg_transform_get_status(void *args)
 867 {
 868     struct wg_transform_get_status_params *params = args;
 869     struct wg_transform *transform = get_transform(params->transform);
 870
 871     params->accepts_input = gst_atomic_queue_length(transform->input_queue) < transform->attrs.input_queue_length + 1;
 872     return STATUS_SUCCESS;
 873 }
 874
 875 NTSTATUS wg_transform_drain(void *args)
 876 {
 877     struct wg_transform *transform = get_transform(*(wg_transform_t *)args);
 878     GstBuffer *input_buffer;
 879     GstFlowReturn ret;
 880     GstEvent *event;
 881
 882     GST_LOG("transform %p", transform);
 883
 884     while ((input_buffer = gst_atomic_queue_pop(transform->input_queue)))
 885     {
 886         if ((ret = gst_pad_push(transform->my_src, input_buffer)))
 887             GST_WARNING("Failed to push transform input, error %d", ret);
 888     }
 889
 890     if (!(event = gst_event_new_segment_done(GST_FORMAT_TIME, -1))
 891             || !push_event(transform->my_src, event))
 892         goto error;
 893     if (!(event = gst_event_new_eos())
 894             || !push_event(transform->my_src, event))
 895         goto error;
 896     if (!(event = gst_event_new_stream_start("stream"))
 897             || !push_event(transform->my_src, event))
 898         goto error;
 899     if (!(event = gst_event_new_segment(&transform->segment))
 900             || !push_event(transform->my_src, event))
 901         goto error;
 902
 903     return STATUS_SUCCESS;
 904
 905 error:
 906     GST_ERROR("Failed to drain transform %p.", transform);
 907     return STATUS_UNSUCCESSFUL;
 908 }
 909
 910 NTSTATUS wg_transform_flush(void *args)
 911 {
 912     struct wg_transform *transform = get_transform(*(wg_transform_t *)args);
 913     GstBuffer *input_buffer;
 914     GstSample *sample;
 915     NTSTATUS status;
 916
 917     GST_LOG("transform %p", transform);
 918
 919     while ((input_buffer = gst_atomic_queue_pop(transform->input_queue)))
 920         gst_buffer_unref(input_buffer);
 921
 922     if ((status = wg_transform_drain(args)))
 923         return status;
 924
 925     while ((sample = gst_atomic_queue_pop(transform->output_queue)))
 926         gst_sample_unref(sample);
 927     if ((sample = transform->output_sample))
 928         gst_sample_unref(sample);
 929     transform->output_sample = NULL;
 930
 931     return STATUS_SUCCESS;
 932 }