modules/hw/d3d11/d3d11_surface.c

   1 /*****************************************************************************
   2  * d3d11_surface.c : D3D11 GPU surface conversion module for vlc
   3  *****************************************************************************
   4  * Copyright © 2015 VLC authors, VideoLAN and VideoLabs
   5  *
   6  * Authors: Steve Lhomme <robux4@gmail.com>
   7  *
   8  * This program is free software; you can redistribute it and/or modify it
   9  * under the terms of the GNU Lesser General Public License as published by
  10  * the Free Software Foundation; either version 2.1 of the License, or
  11  * (at your option) any later version.
  12  *
  13  * This program is distributed in the hope that it will be useful,
  14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
  16  * GNU Lesser General Public License for more details.
  17  *
  18  * You should have received a copy of the GNU Lesser General Public License
  19  * along with this program; if not, write to the Free Software Foundation,
  20  * Inc., 51 Franklin Street, Fifth Floor, Boston MA 02110-1301, USA.
  21  *****************************************************************************/
  22
  23 /*****************************************************************************
  24  * Preamble
  25  *****************************************************************************/
  26
  27 #ifdef HAVE_CONFIG_H
  28 # include "config.h"
  29 #endif
  30
  31 #include <vlc_common.h>
  32 #include <vlc_filter.h>
  33 #include <vlc_picture.h>
  34 #include <vlc_modules.h>
  35
  36 #include <assert.h>
  37
  38 #include "../../video_chroma/copy.h"
  39
  40 #include <windows.h>
  41 #define COBJMACROS
  42 #include <d3d11.h>
  43
  44 #include "d3d11_filters.h"
  45 #include "d3d11_processor.h"
  46 #include "../../video_chroma/d3d11_fmt.h"
  47
  48 #ifdef ID3D11VideoContext_VideoProcessorBlt
  49 #define CAN_PROCESSOR 1
  50 #else
  51 #define CAN_PROCESSOR 0
  52 #endif
  53
  54 typedef struct
  55 {
  56     copy_cache_t     cache;
  57     union {
  58         ID3D11Texture2D  *staging;
  59         ID3D11Resource   *staging_resource;
  60     };
  61     vlc_mutex_t      staging_lock;
  62
  63 #if CAN_PROCESSOR
  64     union {
  65         ID3D11Texture2D  *procOutTexture;
  66         ID3D11Resource   *procOutResource;
  67     };
  68     /* 420_OPAQUE processor */
  69     ID3D11VideoProcessorOutputView *processorOutput;
  70     d3d11_processor_t              d3d_proc;
  71 #endif
  72     d3d11_device_t                 *d3d_dev;
  73
  74     /* CPU to GPU */
  75     filter_t   *filter;
  76     picture_t  *staging_pic;
  77 } filter_sys_t;
  78
  79 #if CAN_PROCESSOR
  80 static int SetupProcessor(filter_t *p_filter, d3d11_device_t *d3d_dev,
  81                           DXGI_FORMAT srcFormat, DXGI_FORMAT dstFormat)
  82 {
  83     filter_sys_t *sys = p_filter->p_sys;
  84     HRESULT hr;
  85
  86     d3d11_device_lock(d3d_dev);
  87
  88     if (D3D11_CreateProcessor(p_filter, d3d_dev, D3D11_VIDEO_FRAME_FORMAT_PROGRESSIVE,
  89                               &p_filter->fmt_in.video, &p_filter->fmt_out.video, &sys->d3d_proc) != VLC_SUCCESS)
  90         goto error;
  91
  92     UINT flags;
  93     /* shortcut for the rendering output */
  94     hr = ID3D11VideoProcessorEnumerator_CheckVideoProcessorFormat(sys->d3d_proc.procEnumerator, srcFormat, &flags);
  95     if (FAILED(hr) || !(flags & D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_INPUT))
  96     {
  97         msg_Dbg(p_filter, "processor format %s not supported for output", DxgiFormatToStr(srcFormat));
  98         goto error;
  99     }
 100     hr = ID3D11VideoProcessorEnumerator_CheckVideoProcessorFormat(sys->d3d_proc.procEnumerator, dstFormat, &flags);
 101     if (FAILED(hr) || !(flags & D3D11_VIDEO_PROCESSOR_FORMAT_SUPPORT_OUTPUT))
 102     {
 103         msg_Dbg(p_filter, "processor format %s not supported for input", DxgiFormatToStr(dstFormat));
 104         goto error;
 105     }
 106
 107     D3D11_VIDEO_PROCESSOR_CAPS processorCaps;
 108     hr = ID3D11VideoProcessorEnumerator_GetVideoProcessorCaps(sys->d3d_proc.procEnumerator, &processorCaps);
 109     for (UINT type = 0; type < processorCaps.RateConversionCapsCount; ++type)
 110     {
 111         hr = ID3D11VideoDevice_CreateVideoProcessor(sys->d3d_proc.d3dviddev,
 112                                                     sys->d3d_proc.procEnumerator, type, &sys->d3d_proc.videoProcessor);
 113         if (SUCCEEDED(hr))
 114         {
 115             D3D11_VIDEO_PROCESSOR_OUTPUT_VIEW_DESC outDesc = {
 116                 .ViewDimension = D3D11_VPOV_DIMENSION_TEXTURE2D,
 117             };
 118
 119             hr = ID3D11VideoDevice_CreateVideoProcessorOutputView(sys->d3d_proc.d3dviddev,
 120                                                              sys->procOutResource,
 121                                                              sys->d3d_proc.procEnumerator,
 122                                                              &outDesc,
 123                                                              &sys->processorOutput);
 124             if (FAILED(hr))
 125                 msg_Err(p_filter, "Failed to create the processor output. (hr=0x%lX)", hr);
 126             else
 127             {
 128                 d3d11_device_unlock(d3d_dev);
 129                 return VLC_SUCCESS;
 130             }
 131         }
 132         if (sys->d3d_proc.videoProcessor)
 133         {
 134             ID3D11VideoProcessor_Release(sys->d3d_proc.videoProcessor);
 135             sys->d3d_proc.videoProcessor = NULL;
 136         }
 137     }
 138
 139 error:
 140     D3D11_ReleaseProcessor(&sys->d3d_proc);
 141     d3d11_device_unlock(d3d_dev);
 142     return VLC_EGENERIC;
 143 }
 144 #endif
 145
 146 static HRESULT can_map(filter_sys_t *sys, ID3D11DeviceContext *context)
 147 {
 148     D3D11_MAPPED_SUBRESOURCE lock;
 149     HRESULT hr = ID3D11DeviceContext_Map(context, sys->staging_resource, 0,
 150                                          D3D11_MAP_READ, 0, &lock);
 151     ID3D11DeviceContext_Unmap(context, sys->staging_resource, 0);
 152     return hr;
 153 }
 154
 155 static int assert_staging(filter_t *p_filter, filter_sys_t *sys, DXGI_FORMAT format)
 156 {
 157     HRESULT hr;
 158
 159     if (sys->staging)
 160         goto ok;
 161
 162     D3D11_TEXTURE2D_DESC texDesc;
 163     ZeroMemory(&texDesc, sizeof(texDesc));
 164     texDesc.Width  = p_filter->fmt_in.video.i_width;
 165     texDesc.Height = p_filter->fmt_in.video.i_height;
 166     texDesc.Format = format;
 167     texDesc.MipLevels = 1;
 168     texDesc.SampleDesc.Count = 1;
 169     texDesc.MiscFlags = 0;
 170     texDesc.ArraySize = 1;
 171     texDesc.Usage = D3D11_USAGE_STAGING;
 172     texDesc.CPUAccessFlags = D3D11_CPU_ACCESS_READ;
 173     texDesc.BindFlags = 0;
 174
 175     d3d11_device_t *d3d_dev = sys->d3d_dev;
 176     sys->staging = NULL;
 177     hr = ID3D11Device_CreateTexture2D( d3d_dev->d3ddevice, &texDesc, NULL, &sys->staging);
 178     /* test if mapping the texture works ref #18746 */
 179     if (SUCCEEDED(hr) && FAILED(hr = can_map(sys, d3d_dev->d3dcontext)))
 180         msg_Dbg(p_filter, "can't map default staging texture (hr=0x%lX)", hr);
 181 #if CAN_PROCESSOR
 182     if (FAILED(hr)) {
 183         /* failed with the this format, try a different one */
 184         UINT supportFlags = D3D11_FORMAT_SUPPORT_SHADER_LOAD | D3D11_FORMAT_SUPPORT_VIDEO_PROCESSOR_OUTPUT;
 185         const d3d_format_t *new_fmt =
 186                 FindD3D11Format( p_filter, d3d_dev, 0, DXGI_RGB_FORMAT|DXGI_YUV_FORMAT, 0, 0, 0, DXGI_CHROMA_CPU, supportFlags );
 187         if (new_fmt && texDesc.Format != new_fmt->formatTexture)
 188         {
 189             DXGI_FORMAT srcFormat = texDesc.Format;
 190             texDesc.Format = new_fmt->formatTexture;
 191             hr = ID3D11Device_CreateTexture2D( d3d_dev->d3ddevice, &texDesc, NULL, &sys->staging);
 192             if (SUCCEEDED(hr))
 193             {
 194                 texDesc.Usage = D3D11_USAGE_DEFAULT;
 195                 texDesc.CPUAccessFlags = 0;
 196                 texDesc.BindFlags |= D3D11_BIND_RENDER_TARGET;
 197                 hr = ID3D11Device_CreateTexture2D( d3d_dev->d3ddevice, &texDesc, NULL, &sys->procOutTexture);
 198                 if (SUCCEEDED(hr) && SUCCEEDED(hr = can_map(sys, d3d_dev->d3dcontext)))
 199                 {
 200                     if (SetupProcessor(p_filter, d3d_dev, srcFormat, new_fmt->formatTexture))
 201                     {
 202                         ID3D11Texture2D_Release(sys->procOutTexture);
 203                         ID3D11Texture2D_Release(sys->staging);
 204                         sys->staging = NULL;
 205                         hr = E_FAIL;
 206                     }
 207                     else
 208                         msg_Dbg(p_filter, "Using shader+processor format %s", new_fmt->name);
 209                 }
 210                 else
 211                 {
 212                     msg_Dbg(p_filter, "can't create intermediate texture (hr=0x%lX)", hr);
 213                     ID3D11Texture2D_Release(sys->staging);
 214                     sys->staging = NULL;
 215                 }
 216             }
 217         }
 218     }
 219 #endif
 220     if (FAILED(hr)) {
 221         msg_Err(p_filter, "Failed to create a %s staging texture to extract surface pixels (hr=0x%lX)", DxgiFormatToStr(texDesc.Format), hr );
 222         return VLC_EGENERIC;
 223     }
 224 ok:
 225     return VLC_SUCCESS;
 226 }
 227
 228 static void D3D11_YUY2(filter_t *p_filter, picture_t *src, picture_t *dst)
 229 {
 230     if (src->context == NULL)
 231     {
 232         /* the previous stages creating a D3D11 picture should always fill the context */
 233         msg_Err(p_filter, "missing source context");
 234         return;
 235     }
 236
 237     filter_sys_t *sys = p_filter->p_sys;
 238     picture_sys_d3d11_t *p_sys = ActiveD3D11PictureSys(src);
 239
 240     D3D11_TEXTURE2D_DESC desc;
 241     D3D11_MAPPED_SUBRESOURCE lock;
 242
 243     vlc_mutex_lock(&sys->staging_lock);
 244     UINT srcSlice = p_sys->slice_index;
 245     ID3D11Resource *srcResource = p_sys->resource[KNOWN_DXGI_INDEX];
 246
 247 #if CAN_PROCESSOR
 248     if (sys->d3d_proc.procEnumerator)
 249     {
 250         HRESULT hr;
 251         if (FAILED( D3D11_Assert_ProcessorInput(p_filter, &sys->d3d_proc, p_sys) ))
 252             return;
 253
 254         D3D11_VIDEO_PROCESSOR_STREAM stream = {
 255             .Enable = TRUE,
 256             .pInputSurface = p_sys->processorInput,
 257         };
 258
 259         hr = ID3D11VideoContext_VideoProcessorBlt(sys->d3d_proc.d3dvidctx, sys->d3d_proc.videoProcessor,
 260                                                           sys->processorOutput,
 261                                                           0, 1, &stream);
 262         if (FAILED(hr))
 263         {
 264             msg_Err(p_filter, "Failed to process the video. (hr=0x%lX)", hr);
 265             vlc_mutex_unlock(&sys->staging_lock);
 266             return;
 267         }
 268
 269         srcResource = sys->procOutResource;
 270         srcSlice = 0;
 271     }
 272 #endif
 273     ID3D11DeviceContext_CopySubresourceRegion(sys->d3d_dev->d3dcontext, sys->staging_resource,
 274                                               0, 0, 0, 0,
 275                                               srcResource,
 276                                               srcSlice,
 277                                               NULL);
 278
 279     HRESULT hr = ID3D11DeviceContext_Map(sys->d3d_dev->d3dcontext, sys->staging_resource,
 280                                          0, D3D11_MAP_READ, 0, &lock);
 281     if (FAILED(hr)) {
 282         msg_Err(p_filter, "Failed to map source surface. (hr=0x%lX)", hr);
 283         vlc_mutex_unlock(&sys->staging_lock);
 284         return;
 285     }
 286
 287     if (dst->format.i_chroma == VLC_CODEC_I420)
 288         picture_SwapUV( dst );
 289
 290     ID3D11Texture2D_GetDesc(sys->staging, &desc);
 291
 292     if (desc.Format == DXGI_FORMAT_YUY2) {
 293         size_t chroma_pitch = (lock.RowPitch / 2);
 294
 295         const size_t pitch[3] = {
 296             lock.RowPitch,
 297             chroma_pitch,
 298             chroma_pitch,
 299         };
 300
 301         const uint8_t *plane[3] = {
 302             (uint8_t*)lock.pData,
 303             (uint8_t*)lock.pData + pitch[0] * desc.Height,
 304             (uint8_t*)lock.pData + pitch[0] * desc.Height
 305                                  + pitch[1] * desc.Height / 2,
 306         };
 307
 308         Copy420_P_to_P(dst, plane, pitch,
 309                        src->format.i_visible_height + src->format.i_y_offset,
 310                        &sys->cache);
 311     } else if (desc.Format == DXGI_FORMAT_NV12 ||
 312                desc.Format == DXGI_FORMAT_P010) {
 313         const uint8_t *plane[2] = {
 314             lock.pData,
 315             (uint8_t*)lock.pData + lock.RowPitch * desc.Height
 316         };
 317         const size_t  pitch[2] = {
 318             lock.RowPitch,
 319             lock.RowPitch,
 320         };
 321         if (desc.Format == DXGI_FORMAT_NV12)
 322             Copy420_SP_to_P(dst, plane, pitch,
 323                             __MIN(desc.Height, src->format.i_y_offset + src->format.i_visible_height),
 324                             &sys->cache);
 325         else
 326             Copy420_16_SP_to_P(dst, plane, pitch,
 327                                __MIN(desc.Height, src->format.i_y_offset + src->format.i_visible_height),
 328                                6, &sys->cache);
 329         picture_SwapUV(dst);
 330     } else {
 331         msg_Err(p_filter, "Unsupported D3D11VA conversion from 0x%08X to YV12", desc.Format);
 332     }
 333
 334     if (dst->format.i_chroma == VLC_CODEC_I420 || dst->format.i_chroma == VLC_CODEC_I420_10L)
 335         picture_SwapUV( dst );
 336
 337     /* */
 338     ID3D11DeviceContext_Unmap(sys->d3d_dev->d3dcontext, sys->staging_resource, 0);
 339     vlc_mutex_unlock(&sys->staging_lock);
 340 }
 341
 342 static void D3D11_NV12(filter_t *p_filter, picture_t *src, picture_t *dst)
 343 {
 344     if (src->context == NULL)
 345     {
 346         /* the previous stages creating a D3D11 picture should always fill the context */
 347         msg_Err(p_filter, "missing source context");
 348         return;
 349     }
 350
 351     filter_sys_t *sys = p_filter->p_sys;
 352     picture_sys_d3d11_t *p_sys = ActiveD3D11PictureSys(src);
 353
 354     D3D11_TEXTURE2D_DESC desc;
 355     D3D11_MAPPED_SUBRESOURCE lock;
 356
 357     vlc_mutex_lock(&sys->staging_lock);
 358     UINT srcSlice = p_sys->slice_index;
 359     ID3D11Resource *srcResource = p_sys->resource[KNOWN_DXGI_INDEX];
 360
 361 #if CAN_PROCESSOR
 362     if (sys->d3d_proc.procEnumerator)
 363     {
 364         HRESULT hr;
 365         if (FAILED( D3D11_Assert_ProcessorInput(p_filter, &sys->d3d_proc, p_sys) ))
 366             return;
 367
 368         D3D11_VIDEO_PROCESSOR_STREAM stream = {
 369             .Enable = TRUE,
 370             .pInputSurface = p_sys->processorInput,
 371         };
 372
 373         hr = ID3D11VideoContext_VideoProcessorBlt(sys->d3d_proc.d3dvidctx, sys->d3d_proc.videoProcessor,
 374                                                           sys->processorOutput,
 375                                                           0, 1, &stream);
 376         if (FAILED(hr))
 377         {
 378             msg_Err(p_filter, "Failed to process the video. (hr=0x%lX)", hr);
 379             vlc_mutex_unlock(&sys->staging_lock);
 380             return;
 381         }
 382
 383         srcResource = sys->procOutResource;
 384         srcSlice = 0;
 385     }
 386 #endif
 387     ID3D11DeviceContext_CopySubresourceRegion(sys->d3d_dev->d3dcontext, sys->staging_resource,
 388                                               0, 0, 0, 0,
 389                                               srcResource,
 390                                               srcSlice,
 391                                               NULL);
 392
 393     HRESULT hr = ID3D11DeviceContext_Map(sys->d3d_dev->d3dcontext, sys->staging_resource,
 394                                          0, D3D11_MAP_READ, 0, &lock);
 395     if (FAILED(hr)) {
 396         msg_Err(p_filter, "Failed to map source surface. (hr=0x%lX)", hr);
 397         vlc_mutex_unlock(&sys->staging_lock);
 398         return;
 399     }
 400
 401     ID3D11Texture2D_GetDesc(sys->staging, &desc);
 402
 403     if (desc.Format == DXGI_FORMAT_NV12 || desc.Format == DXGI_FORMAT_P010) {
 404         const uint8_t *plane[2] = {
 405             lock.pData,
 406             (uint8_t*)lock.pData + lock.RowPitch * desc.Height
 407         };
 408         size_t  pitch[2] = {
 409             lock.RowPitch,
 410             lock.RowPitch,
 411         };
 412         Copy420_SP_to_SP(dst, plane, pitch,
 413                          __MIN(desc.Height, src->format.i_y_offset + src->format.i_visible_height),
 414                          &sys->cache);
 415     } else {
 416         msg_Err(p_filter, "Unsupported D3D11VA conversion from 0x%08X to NV12", desc.Format);
 417     }
 418
 419     /* */
 420     ID3D11DeviceContext_Unmap(sys->d3d_dev->d3dcontext, sys->staging_resource, 0);
 421     vlc_mutex_unlock(&sys->staging_lock);
 422 }
 423
 424 static void D3D11_RGBA(filter_t *p_filter, picture_t *src, picture_t *dst)
 425 {
 426     filter_sys_t *sys = p_filter->p_sys;
 427     assert(src->context != NULL);
 428     picture_sys_d3d11_t *p_sys = ActiveD3D11PictureSys(src);
 429
 430     D3D11_TEXTURE2D_DESC desc;
 431     D3D11_MAPPED_SUBRESOURCE lock;
 432
 433     vlc_mutex_lock(&sys->staging_lock);
 434     ID3D11DeviceContext_CopySubresourceRegion(sys->d3d_dev->d3dcontext, sys->staging_resource,
 435                                               0, 0, 0, 0,
 436                                               p_sys->resource[KNOWN_DXGI_INDEX],
 437                                               p_sys->slice_index,
 438                                               NULL);
 439
 440     HRESULT hr = ID3D11DeviceContext_Map(sys->d3d_dev->d3dcontext, sys->staging_resource,
 441                                          0, D3D11_MAP_READ, 0, &lock);
 442     if (FAILED(hr)) {
 443         msg_Err(p_filter, "Failed to map source surface. (hr=0x%lX)", hr);
 444         vlc_mutex_unlock(&sys->staging_lock);
 445         return;
 446     }
 447
 448     ID3D11Texture2D_GetDesc(sys->staging, &desc);
 449
 450     plane_t src_planes  = dst->p[0];
 451     src_planes.i_lines  = desc.Height;
 452     src_planes.i_pitch  = lock.RowPitch;
 453     src_planes.p_pixels = lock.pData;
 454     plane_CopyPixels( dst->p, &src_planes );
 455
 456     /* */
 457     ID3D11DeviceContext_Unmap(sys->d3d_dev->d3dcontext,
 458                               p_sys->resource[KNOWN_DXGI_INDEX], p_sys->slice_index);
 459     vlc_mutex_unlock(&sys->staging_lock);
 460 }
 461
 462 static void DeleteFilter( filter_t * p_filter )
 463 {
 464     if( p_filter->p_module )
 465     {
 466         filter_Close( p_filter );
 467         module_unneed( p_filter, p_filter->p_module );
 468     }
 469
 470     es_format_Clean( &p_filter->fmt_in );
 471     es_format_Clean( &p_filter->fmt_out );
 472
 473     vlc_object_delete(p_filter);
 474 }
 475
 476 static picture_t *NewBuffer(filter_t *p_filter)
 477 {
 478     filter_t *p_parent = p_filter->owner.sys;
 479     filter_sys_t *p_sys = p_parent->p_sys;
 480     return p_sys->staging_pic;
 481 }
 482
 483 static vlc_decoder_device * HoldD3D11DecoderDevice(vlc_object_t *o, void *sys)
 484 {
 485     VLC_UNUSED(o);
 486     filter_t *p_this = sys;
 487     return filter_HoldDecoderDevice(p_this);
 488 }
 489
 490 static filter_t *CreateCPUtoGPUFilter( filter_t *p_this, const es_format_t *p_fmt_in,
 491                                vlc_fourcc_t dst_chroma )
 492 {
 493     filter_t *p_filter;
 494
 495     p_filter = vlc_object_create( p_this, sizeof(filter_t) );
 496     if (unlikely(p_filter == NULL))
 497         return NULL;
 498
 499     static const struct filter_video_callbacks cbs = { NewBuffer, HoldD3D11DecoderDevice };
 500     p_filter->b_allow_fmt_out_change = false;
 501     p_filter->owner.video = &cbs;
 502     p_filter->owner.sys = p_this;
 503
 504     es_format_InitFromVideo( &p_filter->fmt_in,  &p_fmt_in->video );
 505     es_format_InitFromVideo( &p_filter->fmt_out, &p_fmt_in->video );
 506     p_filter->fmt_out.i_codec = p_filter->fmt_out.video.i_chroma = dst_chroma;
 507     p_filter->p_module = module_need( p_filter, "video converter", NULL, false );
 508
 509     if( !p_filter->p_module )
 510     {
 511         msg_Dbg( p_filter, "no video converter found" );
 512         DeleteFilter( p_filter );
 513         return NULL;
 514     }
 515     assert( p_filter->ops != NULL );
 516
 517     return p_filter;
 518 }
 519
 520 static void NV12_D3D11(filter_t *p_filter, picture_t *src, picture_t *dst)
 521 {
 522     filter_sys_t *sys = p_filter->p_sys;
 523     picture_sys_d3d11_t *p_sys = ActiveD3D11PictureSys(dst);
 524     if (unlikely(p_sys==NULL))
 525     {
 526         /* the output filter configuration may have changed since the filter
 527          * was opened */
 528         return;
 529     }
 530
 531     d3d11_device_lock( sys->d3d_dev );
 532     if (sys->filter == NULL)
 533     {
 534         D3D11_MAPPED_SUBRESOURCE lock;
 535         HRESULT hr = ID3D11DeviceContext_Map(sys->d3d_dev->d3dcontext, p_sys->resource[KNOWN_DXGI_INDEX],
 536                                             0, D3D11_MAP_WRITE_DISCARD, 0, &lock);
 537         if (FAILED(hr)) {
 538             msg_Err(p_filter, "Failed to map source surface. (hr=0x%lX)", hr);
 539             d3d11_device_unlock( sys->d3d_dev );
 540             return;
 541         }
 542
 543         picture_UpdatePlanes(dst, lock.pData, lock.RowPitch);
 544         picture_context_t *dst_pic_ctx = dst->context;
 545         dst->context = NULL; // some CPU filters won't like the mix of CPU/GPU
 546
 547         picture_CopyPixels(dst, src);
 548
 549         dst->context = dst_pic_ctx;
 550         ID3D11DeviceContext_Unmap(sys->d3d_dev->d3dcontext, p_sys->resource[KNOWN_DXGI_INDEX], 0);
 551     }
 552     else
 553     {
 554         picture_sys_d3d11_t *p_staging_sys = p_sys;
 555
 556         D3D11_TEXTURE2D_DESC texDesc;
 557         ID3D11Texture2D_GetDesc( p_staging_sys->texture[KNOWN_DXGI_INDEX], &texDesc);
 558
 559         D3D11_MAPPED_SUBRESOURCE lock;
 560         HRESULT hr = ID3D11DeviceContext_Map(sys->d3d_dev->d3dcontext, p_staging_sys->resource[KNOWN_DXGI_INDEX],
 561                                             0, D3D11_MAP_WRITE_DISCARD, 0, &lock);
 562         if (FAILED(hr)) {
 563             msg_Err(p_filter, "Failed to map source surface. (hr=0x%lX)", hr);
 564             d3d11_device_unlock( sys->d3d_dev );
 565             return;
 566         }
 567
 568         picture_UpdatePlanes(sys->staging_pic, lock.pData, lock.RowPitch);
 569         picture_context_t *staging_pic_ctx = sys->staging_pic->context;
 570         sys->staging_pic->context = NULL; // some CPU filters won't like the mix of CPU/GPU
 571
 572         picture_Hold( src );
 573         sys->filter->ops->filter_video(sys->filter, src);
 574
 575         sys->staging_pic->context = staging_pic_ctx;
 576         ID3D11DeviceContext_Unmap(sys->d3d_dev->d3dcontext, p_staging_sys->resource[KNOWN_DXGI_INDEX], 0);
 577
 578         D3D11_BOX copyBox = {
 579             .right = dst->format.i_width, .bottom = dst->format.i_height, .back = 1,
 580         };
 581         ID3D11DeviceContext_CopySubresourceRegion(sys->d3d_dev->d3dcontext,
 582                                                 p_sys->resource[KNOWN_DXGI_INDEX],
 583                                                 p_sys->slice_index,
 584                                                 0, 0, 0,
 585                                                 p_staging_sys->resource[KNOWN_DXGI_INDEX], 0,
 586                                                 &copyBox);
 587     }
 588     d3d11_device_unlock( sys->d3d_dev );
 589     // stop pretending this is a CPU picture
 590     dst->format.i_chroma = p_filter->fmt_out.video.i_chroma;
 591     dst->i_planes = 0;
 592 }
 593
 594 static void D3D11CloseConverter( filter_t *p_filter )
 595 {
 596     filter_sys_t *p_sys = p_filter->p_sys;
 597 #if CAN_PROCESSOR
 598     if (p_sys->procOutTexture)
 599         ID3D11Texture2D_Release(p_sys->procOutTexture);
 600     D3D11_ReleaseProcessor( &p_sys->d3d_proc );
 601 #endif
 602     CopyCleanCache(&p_sys->cache);
 603     if (p_sys->staging)
 604         ID3D11Texture2D_Release(p_sys->staging);
 605 }
 606
 607 VIDEO_FILTER_WRAPPER_CLOSE(D3D11_NV12, D3D11CloseConverter)
 608 VIDEO_FILTER_WRAPPER_CLOSE(D3D11_YUY2, D3D11CloseConverter)
 609 VIDEO_FILTER_WRAPPER_CLOSE(D3D11_RGBA, D3D11CloseConverter)
 610
 611 static picture_t *AllocateCPUtoGPUTexture(filter_t *p_filter, filter_sys_t *p_sys)
 612 {
 613     video_format_t fmt_staging;
 614
 615     d3d11_video_context_t *vctx_sys = GetD3D11ContextPrivate( p_filter->vctx_out );
 616
 617     const d3d_format_t *cfg = NULL;
 618     for (const d3d_format_t *output_format = DxgiGetRenderFormatList();
 619             output_format->name != NULL; ++output_format)
 620     {
 621         if (output_format->formatTexture == vctx_sys->format &&
 622             !is_d3d11_opaque(output_format->fourcc))
 623         {
 624             cfg = output_format;
 625             break;
 626         }
 627     }
 628     if (unlikely(cfg == NULL))
 629         return NULL;
 630
 631     struct d3d11_pic_context *pic_ctx = calloc(1, sizeof(*pic_ctx));
 632     if (unlikely(pic_ctx == NULL))
 633         goto done;
 634
 635     video_format_Copy(&fmt_staging, &p_filter->fmt_out.video);
 636     fmt_staging.i_chroma = cfg->fourcc;
 637
 638     picture_resource_t dummy_res = {};
 639     picture_t *p_dst = picture_NewFromResource(&fmt_staging, &dummy_res);
 640     if (p_dst == NULL) {
 641         msg_Err(p_filter, "Failed to map create the temporary picture.");
 642         goto done;
 643     }
 644
 645     if (AllocateTextures(p_filter, p_sys->d3d_dev, cfg,
 646                          &p_dst->format, pic_ctx->picsys.texture, p_dst->p) != VLC_SUCCESS)
 647         goto done;
 648
 649     if (unlikely(D3D11_AllocateResourceView(p_filter, p_sys->d3d_dev->d3ddevice, cfg,
 650                                             pic_ctx->picsys.texture, 0, pic_ctx->picsys.renderSrc) != VLC_SUCCESS))
 651         goto done;
 652
 653     pic_ctx->s = (picture_context_t) {
 654         d3d11_pic_context_destroy, d3d11_pic_context_copy,
 655         vlc_video_context_Hold(p_filter->vctx_out),
 656     };
 657     AcquireD3D11PictureSys(&pic_ctx->picsys);
 658     ID3D11Texture2D_Release(pic_ctx->picsys.texture[KNOWN_DXGI_INDEX]);
 659
 660     p_dst->context = &pic_ctx->s;
 661
 662     return p_dst;
 663 done:
 664     free(pic_ctx);
 665     return NULL;
 666 }
 667
 668 static picture_t *NV12_D3D11_Filter( filter_t *p_filter, picture_t *p_pic )
 669 {
 670     picture_t *p_outpic = AllocateCPUtoGPUTexture( p_filter, p_filter->p_sys );
 671     if( p_outpic )
 672     {
 673         NV12_D3D11( p_filter, p_pic, p_outpic );
 674         picture_CopyProperties( p_outpic, p_pic );
 675     }
 676     picture_Release( p_pic );
 677     return p_outpic;
 678 }
 679
 680 static void D3D11CloseCPUConverter( filter_t *p_filter )
 681 {
 682     filter_sys_t *p_sys = p_filter->p_sys;
 683     if (p_sys->filter)
 684         DeleteFilter(p_sys->filter);
 685     if (p_sys->staging_pic)
 686         picture_Release(p_sys->staging_pic);
 687     vlc_video_context_Release(p_filter->vctx_out);
 688 }
 689
 690 static const struct vlc_filter_operations NV12_D3D11_ops = {
 691     .filter_video = NV12_D3D11_Filter, .close = D3D11CloseCPUConverter,
 692 };
 693
 694 int D3D11OpenConverter( filter_t *p_filter )
 695 {
 696     if ( !is_d3d11_opaque(p_filter->fmt_in.video.i_chroma) )
 697         return VLC_EGENERIC;
 698     if ( GetD3D11ContextPrivate(p_filter->vctx_in) == NULL )
 699         return VLC_EGENERIC;
 700
 701     if ( p_filter->fmt_in.video.i_visible_height != p_filter->fmt_out.video.i_visible_height
 702          || p_filter->fmt_in.video.i_width != p_filter->fmt_out.video.i_width )
 703         return VLC_EGENERIC;
 704
 705     uint8_t pixel_bytes = 1;
 706     switch( p_filter->fmt_out.video.i_chroma ) {
 707     case VLC_CODEC_I420:
 708     case VLC_CODEC_YV12:
 709         if( p_filter->fmt_in.video.i_chroma != VLC_CODEC_D3D11_OPAQUE )
 710             return VLC_EGENERIC;
 711         p_filter->ops = &D3D11_YUY2_ops;
 712         break;
 713     case VLC_CODEC_I420_10L:
 714         if( p_filter->fmt_in.video.i_chroma != VLC_CODEC_D3D11_OPAQUE_10B )
 715             return VLC_EGENERIC;
 716         p_filter->ops = &D3D11_YUY2_ops;
 717         pixel_bytes = 2;
 718         break;
 719     case VLC_CODEC_NV12:
 720         if( p_filter->fmt_in.video.i_chroma != VLC_CODEC_D3D11_OPAQUE )
 721             return VLC_EGENERIC;
 722         p_filter->ops = &D3D11_NV12_ops;
 723         break;
 724     case VLC_CODEC_P010:
 725         if( p_filter->fmt_in.video.i_chroma != VLC_CODEC_D3D11_OPAQUE_10B )
 726             return VLC_EGENERIC;
 727         p_filter->ops = &D3D11_NV12_ops;
 728         pixel_bytes = 2;
 729         break;
 730     case VLC_CODEC_RGBA:
 731         if( p_filter->fmt_in.video.i_chroma != VLC_CODEC_D3D11_OPAQUE_RGBA )
 732             return VLC_EGENERIC;
 733         p_filter->ops = &D3D11_RGBA_ops;
 734         break;
 735     case VLC_CODEC_BGRA:
 736         if( p_filter->fmt_in.video.i_chroma != VLC_CODEC_D3D11_OPAQUE_BGRA )
 737             return VLC_EGENERIC;
 738         p_filter->ops = &D3D11_RGBA_ops;
 739         break;
 740     default:
 741         return VLC_EGENERIC;
 742     }
 743
 744     filter_sys_t *p_sys = vlc_obj_calloc(VLC_OBJECT(p_filter), 1, sizeof(filter_sys_t));
 745     if (!p_sys)
 746         return VLC_ENOMEM;
 747
 748     d3d11_video_context_t *vctx_sys = GetD3D11ContextPrivate(p_filter->vctx_in);
 749     d3d11_decoder_device_t *dev_sys = GetD3D11OpaqueContext(p_filter->vctx_in);
 750     p_sys->d3d_dev = &dev_sys->d3d_dev;
 751
 752     if (assert_staging(p_filter, p_sys, vctx_sys->format) != VLC_SUCCESS)
 753     {
 754         return VLC_EGENERIC;
 755     }
 756
 757     if (CopyInitCache(&p_sys->cache, p_filter->fmt_in.video.i_width * pixel_bytes))
 758         return VLC_ENOMEM;
 759
 760     vlc_mutex_init(&p_sys->staging_lock);
 761     p_filter->p_sys = p_sys;
 762     return VLC_SUCCESS;
 763 }
 764
 765 int D3D11OpenCPUConverter( filter_t *p_filter )
 766 {
 767     int err = VLC_EGENERIC;
 768     filter_sys_t *p_sys = NULL;
 769
 770     if ( p_filter->fmt_out.video.i_chroma != VLC_CODEC_D3D11_OPAQUE
 771      &&  p_filter->fmt_out.video.i_chroma != VLC_CODEC_D3D11_OPAQUE_10B )
 772         return VLC_EGENERIC;
 773
 774     if ( p_filter->fmt_in.video.i_height != p_filter->fmt_out.video.i_height
 775          || p_filter->fmt_in.video.i_width != p_filter->fmt_out.video.i_width )
 776         return VLC_EGENERIC;
 777
 778     switch( p_filter->fmt_in.video.i_chroma ) {
 779     case VLC_CODEC_I420:
 780     case VLC_CODEC_I420_10L:
 781     case VLC_CODEC_YV12:
 782     case VLC_CODEC_NV12:
 783     case VLC_CODEC_P010:
 784         p_filter->ops = &NV12_D3D11_ops;
 785         break;
 786     default:
 787         return VLC_EGENERIC;
 788     }
 789
 790     vlc_decoder_device *dec_device = filter_HoldDecoderDeviceType( p_filter, VLC_DECODER_DEVICE_D3D11VA );
 791     if (dec_device == NULL)
 792     {
 793         msg_Err(p_filter, "Missing decoder device");
 794         return VLC_EGENERIC;
 795     }
 796     d3d11_decoder_device_t *devsys = GetD3D11OpaqueDevice(dec_device);
 797     if (unlikely(devsys == NULL))
 798     {
 799         msg_Err(p_filter, "Incompatible decoder device %d", dec_device->type);
 800         vlc_decoder_device_Release(dec_device);
 801         return VLC_EGENERIC;
 802     }
 803
 804     p_sys = vlc_obj_calloc(VLC_OBJECT(p_filter), 1, sizeof(filter_sys_t));
 805     if (!p_sys) {
 806         vlc_decoder_device_Release(dec_device);
 807         return VLC_ENOMEM;
 808     }
 809     p_sys->d3d_dev = &devsys->d3d_dev;
 810
 811     DXGI_FORMAT vctx_fmt;
 812     switch( p_filter->fmt_in.video.i_chroma ) {
 813     case VLC_CODEC_I420:
 814     case VLC_CODEC_YV12:
 815     case VLC_CODEC_NV12:
 816         vctx_fmt = DXGI_FORMAT_NV12;
 817         break;
 818     case VLC_CODEC_I420_10L:
 819     case VLC_CODEC_P010:
 820         vctx_fmt = DXGI_FORMAT_P010;
 821         break;
 822     default:
 823         vlc_assert_unreachable();
 824     }
 825     p_filter->vctx_out = D3D11CreateVideoContext(dec_device, vctx_fmt);
 826     if ( p_filter->vctx_out == NULL )
 827     {
 828         msg_Dbg(p_filter, "no video context");
 829         goto done;
 830     }
 831
 832     vlc_fourcc_t d3d_fourcc = DxgiFormatFourcc(vctx_fmt);
 833
 834     if ( p_filter->fmt_in.video.i_chroma != d3d_fourcc )
 835     {
 836         p_sys->staging_pic = AllocateCPUtoGPUTexture(p_filter, p_sys);
 837         if (p_sys->staging_pic == NULL)
 838             goto done;
 839
 840         p_sys->filter = CreateCPUtoGPUFilter(p_filter, &p_filter->fmt_in, d3d_fourcc);
 841         if (!p_sys->filter)
 842         {
 843             picture_Release(p_sys->staging_pic);
 844             goto done;
 845         }
 846     }
 847
 848     p_filter->p_sys = p_sys;
 849     err = VLC_SUCCESS;
 850
 851 done:
 852     if (err != VLC_SUCCESS)
 853     {
 854         vlc_video_context_Release(p_filter->vctx_out);
 855     }
 856     return err;
 857 }