From f7364dfb4b203b74700eead1836e208a49872244 Mon Sep 17 00:00:00 2001
From: Andrew Eikum <aeikum@codeweavers.com>
Date: Wed, 28 Sep 2011 13:31:40 -0500
Subject: [PATCH] mmdevapi: Don't fail if dwChannelMask is not set correctly.

---
 dlls/mmdevapi/tests/render.c      | 41 +++++++++++++++++++++++++++++++++++++++
 dlls/winealsa.drv/mmdevdrv.c      |  1 +
 dlls/winecoreaudio.drv/mmdevdrv.c |  6 ++++++
 dlls/wineoss.drv/mmdevdrv.c       |  9 +++++----
 4 files changed, 53 insertions(+), 4 deletions(-)

diff --git a/dlls/mmdevapi/tests/render.c b/dlls/mmdevapi/tests/render.c
index ce5247db4d8..494c00d7c56 100644
--- a/dlls/mmdevapi/tests/render.c
+++ b/dlls/mmdevapi/tests/render.c
@@ -220,6 +220,47 @@ static void test_audioclient(void)
             NULL, (void**)&ac);
     ok(hr == S_OK, "Activation failed with %08x\n", hr);
 
+    if(pwfx->wFormatTag == WAVE_FORMAT_EXTENSIBLE){
+        WAVEFORMATEXTENSIBLE *fmtex = (WAVEFORMATEXTENSIBLE*)pwfx;
+        WAVEFORMATEX *fmt2 = NULL;
+
+        ok(fmtex->dwChannelMask != 0, "Got empty dwChannelMask\n");
+
+        fmtex->dwChannelMask = 0xffff;
+
+        hr = IAudioClient_Initialize(ac, AUDCLNT_SHAREMODE_SHARED, 0, 5000000, 0, pwfx, NULL);
+        ok(hr == S_OK, "Initialize(dwChannelMask = 0xffff) returns %08x\n", hr);
+
+        IAudioClient_Release(ac);
+
+        hr = IMMDevice_Activate(dev, &IID_IAudioClient, CLSCTX_INPROC_SERVER,
+                NULL, (void**)&ac);
+        ok(hr == S_OK, "Activation failed with %08x\n", hr);
+
+        fmtex->dwChannelMask = 0;
+
+        hr = IAudioClient_IsFormatSupported(ac, AUDCLNT_SHAREMODE_SHARED, pwfx, &fmt2);
+        ok(hr == S_OK, "IsFormatSupported(dwChannelMask = 0) call returns %08x\n", hr);
+        ok(fmtex->dwChannelMask == 0, "Passed format was modified\n");
+
+        CoTaskMemFree(fmt2);
+
+        hr = IAudioClient_Initialize(ac, AUDCLNT_SHAREMODE_SHARED, 0, 5000000, 0, pwfx, NULL);
+        ok(hr == S_OK, "Initialize(dwChannelMask = 0) returns %08x\n", hr);
+
+        IAudioClient_Release(ac);
+
+        hr = IMMDevice_Activate(dev, &IID_IAudioClient, CLSCTX_INPROC_SERVER,
+                NULL, (void**)&ac);
+        ok(hr == S_OK, "Activation failed with %08x\n", hr);
+
+        CoTaskMemFree(pwfx);
+
+        hr = IAudioClient_GetMixFormat(ac, &pwfx);
+        ok(hr == S_OK, "Valid GetMixFormat returns %08x\n", hr);
+    }else
+        skip("Skipping dwChannelMask tests\n");
+
     hr = IAudioClient_Initialize(ac, AUDCLNT_SHAREMODE_SHARED, 0, 5000000, 0, pwfx, NULL);
     ok(hr == S_OK, "Valid Initialize returns %08x\n", hr);
 
diff --git a/dlls/winealsa.drv/mmdevdrv.c b/dlls/winealsa.drv/mmdevdrv.c
index af367ba94a6..e1f1a49424e 100644
--- a/dlls/winealsa.drv/mmdevdrv.c
+++ b/dlls/winealsa.drv/mmdevdrv.c
@@ -1383,6 +1383,7 @@ static HRESULT WINAPI AudioClient_IsFormatSupported(IAudioClient *iface,
         ((WAVEFORMATEXTENSIBLE*)closest)->dwChannelMask = mask;
 
         if(fmt->wFormatTag == WAVE_FORMAT_EXTENSIBLE &&
+                fmtex->dwChannelMask != 0 &&
                 fmtex->dwChannelMask != mask)
             hr = S_FALSE;
     }
diff --git a/dlls/winecoreaudio.drv/mmdevdrv.c b/dlls/winecoreaudio.drv/mmdevdrv.c
index be4ae84ba2c..26c154fb932 100644
--- a/dlls/winecoreaudio.drv/mmdevdrv.c
+++ b/dlls/winecoreaudio.drv/mmdevdrv.c
@@ -1118,6 +1118,7 @@ static HRESULT WINAPI AudioClient_IsFormatSupported(IAudioClient *iface,
         WAVEFORMATEX **outpwfx)
 {
     ACImpl *This = impl_from_IAudioClient(iface);
+    WAVEFORMATEXTENSIBLE *fmtex = (WAVEFORMATEXTENSIBLE*)pwfx;
     AudioQueueRef aqueue;
     HRESULT hr;
 
@@ -1135,6 +1136,11 @@ static HRESULT WINAPI AudioClient_IsFormatSupported(IAudioClient *iface,
 
     dump_fmt(pwfx);
 
+    if(pwfx->wFormatTag == WAVE_FORMAT_EXTENSIBLE &&
+            fmtex->dwChannelMask != 0 &&
+            fmtex->dwChannelMask != get_channel_mask(pwfx->nChannels))
+        return AUDCLNT_E_UNSUPPORTED_FORMAT;
+
     OSSpinLockLock(&This->lock);
 
     hr = ca_setup_aqueue(This->adevid, This->dataflow, pwfx, NULL, &aqueue);
diff --git a/dlls/wineoss.drv/mmdevdrv.c b/dlls/wineoss.drv/mmdevdrv.c
index c34d8095c21..8d1a99747f8 100644
--- a/dlls/wineoss.drv/mmdevdrv.c
+++ b/dlls/wineoss.drv/mmdevdrv.c
@@ -668,7 +668,7 @@ static WAVEFORMATEX *clone_format(const WAVEFORMATEX *fmt)
 }
 
 static HRESULT setup_oss_device(ACImpl *This, const WAVEFORMATEX *fmt,
-        WAVEFORMATEX **out)
+        WAVEFORMATEX **out, BOOL query)
 {
     int tmp, oss_format;
     double tenth;
@@ -723,7 +723,8 @@ static HRESULT setup_oss_device(ACImpl *This, const WAVEFORMATEX *fmt,
 
         ((WAVEFORMATEXTENSIBLE*)closest)->dwChannelMask = mask;
 
-        if(fmt->wFormatTag == WAVE_FORMAT_EXTENSIBLE &&
+        if(query && fmt->wFormatTag == WAVE_FORMAT_EXTENSIBLE &&
+                fmtex->dwChannelMask != 0 &&
                 fmtex->dwChannelMask != mask)
             ret = S_FALSE;
     }
@@ -861,7 +862,7 @@ static HRESULT WINAPI AudioClient_Initialize(IAudioClient *iface,
         return AUDCLNT_E_ALREADY_INITIALIZED;
     }
 
-    hr = setup_oss_device(This, fmt, NULL);
+    hr = setup_oss_device(This, fmt, NULL, FALSE);
     if(hr == S_FALSE){
         LeaveCriticalSection(&This->lock);
         return AUDCLNT_E_UNSUPPORTED_FORMAT;
@@ -1099,7 +1100,7 @@ static HRESULT WINAPI AudioClient_IsFormatSupported(IAudioClient *iface,
 
     EnterCriticalSection(&This->lock);
 
-    ret = setup_oss_device(This, pwfx, outpwfx);
+    ret = setup_oss_device(This, pwfx, outpwfx, TRUE);
 
     LeaveCriticalSection(&This->lock);
 
-- 
2.11.4.GIT