myvtm/Other/libaudioframework/audiomicspk2.c

#include "precompile.h"
#include "audiomicspk.h"
#include "audiomicspk2.h"
#include "audiocontext.h"

#include <assert.h>

#include "audiolog.h"
#include "./other/delaybuf.h"

#include <dmo.h>
#include <Mmsystem.h>
#include <objbase.h>
#include <mediaobj.h>
#include <uuids.h>
#include <propidl.h>
#include <wmcodecdsp.h>

#include <audioclient.h>
#include <MMDeviceApi.h>
#include <AudioEngineEndPoint.h>
#include <DeviceTopology.h>
#include <EndpointVolume.h>
#include <functiondiscoverykeys.h>

#include <portaudio.h>

#include <speex/speex_resampler.h>

#include <math.h>
#define CLOCK_PERIOD	20
#define MAX_STR_LEN 512
#define AUDIO_CLOCK	8000
#define CAPTURE_AUDIO_CLOCK	16000
#define MAX_DELAY	100
#define MS_REF_TICK	10000
#define RESAMPLE_QUALITY	6

#ifndef M_PI
#define M_PI              (3.14159265)
#endif
#define TWOPI             (M_PI * 2.0)

// play use portaudio

typedef struct tagAUDIO_DEVICE_INFO
{
	char szDeviceName[MAX_STR_LEN];
	char szDeviceID[MAX_STR_LEN];
} AUDIO_DEVICE_INFO, *PAUDIO_DEVICE_INFO;

static HRESULT GetDeviceNum(EDataFlow eDataFlow, UINT *uDevCount)
{
	IMMDeviceEnumerator *pEnumerator = NULL;
	IMMDeviceCollection *pEndpoints = NULL;
	HRESULT hr;

	hr = CoCreateInstance(&CLSID_MMDeviceEnumerator, NULL, CLSCTX_INPROC_SERVER, &IID_IMMDeviceEnumerator, (void**)&pEnumerator);
	if (FAILED(hr))
		goto on_error;
	hr = pEnumerator->lpVtbl->EnumAudioEndpoints(pEnumerator, eDataFlow, DEVICE_STATE_ACTIVE, (void**)&pEndpoints);
	if (FAILED(hr))
		goto on_error;
	hr = pEndpoints->lpVtbl->GetCount(pEndpoints, uDevCount);
	if (FAILED(hr))
		goto on_error;

on_error:
	if (pEndpoints)
		pEndpoints->lpVtbl->Release(pEndpoints);
	if (pEnumerator)
		pEnumerator->lpVtbl->Release(pEnumerator);
	return hr;
}

static HRESULT EnumDevice(EDataFlow eDataFlow, UINT uNumElements, AUDIO_DEVICE_INFO *pDevicInfo)
{
	IMMDeviceEnumerator *pEnumerator = NULL;
	IMMDeviceCollection *pEndpoints = NULL;
	HRESULT hr;
	UINT uCount;
	UINT uIdx;

	hr = CoCreateInstance(&CLSID_MMDeviceEnumerator, NULL, CLSCTX_INPROC_SERVER, &IID_IMMDeviceEnumerator, (void**)&pEnumerator);
	if (FAILED(hr))
		goto on_error;
	hr = pEnumerator->lpVtbl->EnumAudioEndpoints(pEnumerator, eDataFlow, DEVICE_STATE_ACTIVE, (void**)&pEndpoints);
	if (FAILED(hr))
		goto on_error;
	hr = pEndpoints->lpVtbl->GetCount(pEndpoints, &uCount);
	if (FAILED(hr))
		goto on_error;

	ZeroMemory(pDevicInfo, sizeof(AUDIO_DEVICE_INFO)*uNumElements);

	for (uIdx = 0; uIdx < uCount && uIdx < uNumElements; ++uIdx) {
		IMMDevice *pDevice = NULL;
		IPropertyStore *pPS = NULL;
		WCHAR* pszDeviceId = NULL;
		PROPVARIANT value;
		PropVariantInit(&value);
		pEndpoints->lpVtbl->Item(pEndpoints, uIdx, &pDevice);
		pDevice->lpVtbl->GetId(pDevice, &pszDeviceId);
		pDevice->lpVtbl->OpenPropertyStore(pDevice, STGM_READ, &pPS);
		pPS->lpVtbl->GetValue(pPS, &PKEY_Device_FriendlyName, &value);
		WideCharToMultiByte(CP_ACP, 0, pszDeviceId, -1, pDevicInfo[uIdx].szDeviceID, MAX_STR_LEN-1, NULL, NULL);
		WideCharToMultiByte(CP_ACP, 0, value.pwszVal, -1, pDevicInfo[uIdx].szDeviceName, MAX_STR_LEN-1, NULL, NULL);
		PropVariantClear(&value);
		CoTaskMemFree(pszDeviceId);
		pPS->lpVtbl->Release(pPS);
		pDevice->lpVtbl->Release(pDevice);
	}

on_error:
	if (pEndpoints)
		pEndpoints->lpVtbl->Release(pEndpoints);
	if (pEnumerator)
		pEnumerator->lpVtbl->Release(pEnumerator);
	return hr;
}

static HRESULT DeviceBindTo(
	EDataFlow eDataFlow,    // eCapture/eRender
	INT iDevIdx,        // Device Index. -1 - default device. 
	IAudioClient **ppAudioClient,    // pointer pointer to IAudioClient interface
	IAudioEndpointVolume **ppEndpointVolume
	)
{
	IMMDeviceEnumerator *pEnumerator = NULL;
	IMMDeviceCollection *pEndpoints = NULL;
	IMMDevice *pDevice = NULL;
	HRESULT hr;

	hr = CoCreateInstance(&CLSID_MMDeviceEnumerator, NULL, CLSCTX_INPROC_SERVER, &IID_IMMDeviceEnumerator, (void**)&pEnumerator);
	if (FAILED(hr))
		goto on_error;

	if (iDevIdx < 0) {
		hr = pEnumerator->lpVtbl->GetDefaultAudioEndpoint(pEnumerator, eDataFlow, eConsole, &pDevice);
		if (FAILED(hr))
			goto on_error;
	} else {
		hr = pEnumerator->lpVtbl->EnumAudioEndpoints(pEnumerator, eDataFlow, DEVICE_STATE_ACTIVE, (void**)&pEndpoints);
		if (FAILED(hr))
			goto on_error;
		hr = pEndpoints->lpVtbl->Item(pEndpoints, iDevIdx, &pDevice);
		if (FAILED(hr))
			goto on_error;
	}

	hr = pDevice->lpVtbl->Activate(pDevice, &IID_IAudioClient, CLSCTX_INPROC_SERVER, NULL, (void**)ppAudioClient);
	if (FAILED(hr))
		goto on_error;

	if (ppEndpointVolume) {
		hr = pDevice->lpVtbl->Activate(pDevice, &IID_IAudioEndpointVolume, CLSCTX_INPROC_SERVER, NULL, (void**)ppEndpointVolume);
		if (FAILED(hr)) {
			if (*ppAudioClient) {
				(*ppAudioClient)->lpVtbl->Release(*ppAudioClient);
				*ppAudioClient = NULL;
			}
			goto on_error;
		}
	}

on_error:

	if (pDevice) {
		pDevice->lpVtbl->Release(pDevice);
	}
	if (pEndpoints) {
		pEndpoints->lpVtbl->Release(pEndpoints);
	}
	if (pEnumerator) {
		pEnumerator->lpVtbl->Release(pEnumerator);
	}

	return hr;
}

static int get_device_index(int indev, const char *key)
{
	EDataFlow df = indev ? eCapture : eRender;
	UINT i;
	UINT n;
	AUDIO_DEVICE_INFO *pInfo = NULL;
	GetDeviceNum(df, &n);
	pInfo = malloc(sizeof(AUDIO_DEVICE_INFO) * n);
	EnumDevice(df, n, pInfo);
	for (i = 0; i < n; ++i) {
		if (strstr(pInfo[i].szDeviceName, key)) {
			free(pInfo);
			return i;
		}
	}
	free(pInfo);
	return -1;
}

static int get_device_index_poartaudio(int indev, const char *key)
{
	int i;
	int n = Pa_GetDeviceCount();
	for (i = 0; i < n; ++i) {
		const PaDeviceInfo* pInfo = Pa_GetDeviceInfo(i);
		if (indev) {
			if (pInfo->maxInputChannels && strstr(pInfo->name, key))
				return i;
		} else {
			if (pInfo->maxOutputChannels && strstr(pInfo->name, key))
				return i;
		}
	}
	return -1;
}

static HRESULT CreateVoiceCaptureDMO(
	int spk_dev_id,
	int rec_dev_id,
	BOOL bNS,
	BOOL bAGC, 
	IMediaObject *ppDMO)
{
	HRESULT hr = S_OK;
	return hr;
}

typedef struct IMediaBufferImpl {
	struct IMediaBufferVtbl *lpVtbl;
	struct IMediaBufferVtbl vtbl;
	BYTE *m_pData;
	ULONG m_ulSize;
	ULONG m_ulData;
	ULONG m_cRef;
}IMediaBufferImpl;

static HRESULT STDMETHODCALLTYPE QueryInterface(IMediaBuffer * This, REFIID riid, void** ppvObject);
static ULONG STDMETHODCALLTYPE AddRef(IMediaBuffer * This);
static ULONG STDMETHODCALLTYPE Release(IMediaBuffer * This);
static HRESULT STDMETHODCALLTYPE SetLength(IMediaBuffer * This, DWORD ulLength);
static HRESULT STDMETHODCALLTYPE GetMaxLength(IMediaBuffer * This, DWORD *pcbMaxLength);
static HRESULT STDMETHODCALLTYPE GetBufferAndLength(IMediaBuffer * This, BYTE **ppBuffer, DWORD *pcbLength);

static void destroy_media_buffer(IMediaBufferImpl *pImpl)
{
	free(pImpl->m_pData);
	free(pImpl);
}

static IMediaBuffer *create_media_buffer(ULONG len)
{
	IMediaBufferImpl *pImpl = (IMediaBufferImpl*)malloc(sizeof(IMediaBufferImpl));
	if (pImpl) {
		pImpl->lpVtbl = &pImpl->vtbl;
		pImpl->lpVtbl->QueryInterface = &QueryInterface;
		pImpl->lpVtbl->AddRef = &AddRef;
		pImpl->lpVtbl->Release = &Release;
		pImpl->lpVtbl->SetLength = &SetLength;
		pImpl->lpVtbl->GetMaxLength = &GetMaxLength;
		pImpl->lpVtbl->GetBufferAndLength = &GetBufferAndLength;
		pImpl->m_pData = (BYTE*)malloc(len);
		pImpl->m_ulData = 0;
		pImpl->m_ulSize = len;
		pImpl->m_cRef = 1;
	}
	return (IMediaBuffer*)pImpl;
}

static HRESULT STDMETHODCALLTYPE QueryInterface(IMediaBuffer * This, REFIID riid, void** ppvObject)
{
	if (This == NULL || riid == NULL || ppvObject == NULL || *ppvObject == NULL)
		return E_POINTER;
	if (IsEqualIID(riid, &IID_IUnknown)) {
		*ppvObject = (void*)This;
		return S_OK;
	} else if (IsEqualIID(riid, &IID_IMediaBuffer)) {
		*ppvObject = (void*)This;
		return S_OK;
	}
	return E_NOTIMPL;
}
static ULONG STDMETHODCALLTYPE AddRef(IMediaBuffer * This)
{
	IMediaBufferImpl *pImpl = (IMediaBufferImpl *)This;
	return InterlockedIncrement(&pImpl->m_cRef);
}
static ULONG STDMETHODCALLTYPE Release(IMediaBuffer * This)
{
	IMediaBufferImpl *pImpl = (IMediaBufferImpl *)This;
	ULONG lRet = InterlockedDecrement(&pImpl->m_cRef);
	if (lRet == 0) {
		destroy_media_buffer(pImpl);
	}
	return lRet;
}
static HRESULT STDMETHODCALLTYPE SetLength(IMediaBuffer * This, DWORD ulLength) 
{
	IMediaBufferImpl *pImpl = (IMediaBufferImpl *)This;
	pImpl->m_ulData = ulLength; 
	return NOERROR;
}
static HRESULT STDMETHODCALLTYPE GetMaxLength(IMediaBuffer * This, DWORD *pcbMaxLength) 
{
	IMediaBufferImpl *pImpl = (IMediaBufferImpl *)This;
	*pcbMaxLength = pImpl->m_ulSize; 
	return NOERROR;
}
static HRESULT STDMETHODCALLTYPE GetBufferAndLength(IMediaBuffer * This, BYTE **ppBuffer, DWORD *pcbLength) 
{
	IMediaBufferImpl *pImpl = (IMediaBufferImpl *)This;
	if (ppBuffer) *ppBuffer = pImpl->m_pData;
	if (pcbLength) *pcbLength = pImpl->m_ulData;
	return NOERROR;
}



static int load_data(audiomicspk2_t *micspk, BYTE *pData, UINT32 nCount, UINT32 *pReturnedCount)
{
	UINT32 frame_samples = AUDIO_CLOCK * FRAME_TIME / 1000;
	UINT32 frame_offset = 0;
	while (nCount >= frame_samples) {
		delay_buf_get(micspk->ply_dbuf, (SHORT*)(pData+frame_offset*2));
		frame_offset += frame_samples;
		nCount -= frame_samples;
	}
	*pReturnedCount = frame_offset;
	return 0;
}

static int save_data(audiomicspk2_t *micspk, IMediaBuffer *pBuffer)
{
	UINT32 frame_samples = AUDIO_CLOCK * FRAME_TIME / 1000;
	UINT32 capture_frame_samples = CAPTURE_AUDIO_CLOCK * FRAME_TIME / 1000;
	UINT32 frame_offset = 0;
	BYTE *pData;
	DWORD dwLength;
	pBuffer->lpVtbl->GetBufferAndLength(pBuffer, &pData, &dwLength);
	//AUDIO_LOG_INFO("micspk2 rec samples:%d", dwLength>>1);
	while (dwLength >= capture_frame_samples*2) {
		spx_uint32_t in_len = capture_frame_samples;
		spx_uint32_t out_len = frame_samples;
		SHORT *pFrameData = (SHORT*)(pData+frame_offset*2);
		SHORT outData[160];
		speex_resampler_process_int((SpeexResamplerState*)micspk->input_resample_state, 0, 
			pFrameData, &in_len, outData, &out_len);
		delay_buf_put((delay_buf*)micspk->rec_dbuf, outData);
		frame_offset += capture_frame_samples;
		dwLength -= capture_frame_samples*2;
	}
	return 0;
}

typedef struct paTestData
{
	int    sleepFor;
	double phase;
}
paTestData;
static int StreamCallback(const void *input, 
	void *output, 
	unsigned long frameCount, 
	const PaStreamCallbackTimeInfo* timeInfo, 
	PaStreamCallbackFlags statusFlags, 
	void *userData)
{
	audiomicspk2_t *micspk = (audiomicspk2_t*)userData;
	apr_status_t status;
				//double phase = 0;
				//int i=0;
	if (output) {
		unsigned nsamples_req = frameCount;
		if (micspk->ply_buf_cnt == 0 && nsamples_req == micspk->capture_frame_samples) {
			SHORT data[160];
			spx_uint32_t in_len = micspk->frame_samples;
			spx_uint32_t out_len = micspk->capture_frame_samples;
			delay_buf_get((delay_buf*)micspk->ply_dbuf, data);
			if (micspk->user_data/*&&strlen(data)*/)
			{
				micspk->on_rx_audio((char*)data,micspk->user_data);
			}	
			speex_resampler_process_int((SpeexResamplerState*)micspk->output_resample_state, 0, 
				data, &in_len, (spx_int16_t*)output, &out_len);
		}
	}
	//paTestData *data = (paTestData*)userData;
	//float *out = (float*)output;
	//unsigned long i;
	//int finished = 0;
	//double phaseInc = 0.02;
	//double phase = data->phase;

	//(void) input; /* Prevent unused argument warning. */

	//for( i=0; i<frameCount; i++ )
	//{
	//	phase += phaseInc;
	//	if( phase > TWOPI ) phase -= TWOPI;
	//	/* This is not a very efficient way to calc sines. */
	//	*out++ = (float) sin( phase )*2; /* mono */
	//}
	//write_log("output=%d",(spx_int16_t*)output);
	//if( data->sleepFor > 0 )
	//{
	//	Pa_Sleep(1000 );
	//}

	//data->phase = phase;
	return paContinue;
}

static int initialize_spk(audiomicspk2_t *micspk)
{
	UINT32 capture_frame_samples = CAPTURE_AUDIO_CLOCK * FRAME_TIME / 1000;
	UINT32 frame_samples = AUDIO_CLOCK * FRAME_TIME / 1000;
	const PaDeviceInfo *info;
	PaStreamParameters outParam;
	PaError paError;
	int ply_dev_id = micspk->ply_dev_id_portaudio;

	if (ply_dev_id == -1) {
		ply_dev_id = Pa_GetDefaultOutputDevice();
		if (ply_dev_id == paNoDevice) {
			AUDIO_LOG_ERROR("audiomicspk create error, cannot find output device");
			return APR_EGENERAL;
		}
	}
	info = Pa_GetDeviceInfo(ply_dev_id);
	outParam.device = ply_dev_id;
	outParam.channelCount = 1;
	outParam.sampleFormat = paInt16;
	outParam.suggestedLatency = info->defaultLowOutputLatency;
	outParam.hostApiSpecificStreamInfo = NULL;
	if (Pa_IsFormatSupported(NULL, &outParam, AUDIO_CLOCK) != paNoError) {
		AUDIO_LOG_ERROR("audiomicspk create error, cannot open audio output device");
		return APR_EGENERAL;
	}

	paError = Pa_OpenStream(&micspk->ply_stream, NULL, &outParam, CAPTURE_AUDIO_CLOCK, capture_frame_samples, paClipOff, &StreamCallback, micspk);
	if (paError != 0) {
		AUDIO_LOG_ERROR("audiomicspk create error, Pa_OpenStream function failed in dir! ");
		return APR_EGENERAL;
	}
	paError = Pa_StartStream(micspk->ply_stream);
	if (paError != 0) {
		AUDIO_LOG_ERROR("Pa_StartStream function failed in dir! ");
		Pa_CloseStream(micspk->ply_stream);
		micspk->ply_stream = NULL;
		return APR_EGENERAL;
	}

	return 0;
}

static int initialize_rec(audiomicspk2_t *micspk)
{
	IMediaObject *pDMO = NULL;
	IPropertyStore *pPS = NULL;
	HRESULT hr;
	UINT32 capture_frame_samples = CAPTURE_AUDIO_CLOCK * FRAME_TIME / 1000;
	UINT32 frame_samples = AUDIO_CLOCK * FRAME_TIME / 1000;

	hr = CoCreateInstance(&CLSID_CWMAudioAEC, NULL, CLSCTX_INPROC_SERVER, &IID_IMediaObject, (void**)&pDMO);

	if (SUCCEEDED(hr)) {
		hr = pDMO->lpVtbl->QueryInterface(pDMO, &IID_IPropertyStore, (void**)&pPS);
	}

	if (FAILED(hr)) {
		if (pDMO) {
			pDMO->lpVtbl->Release(pDMO);
		}
		return -1;
	}

	// sys mode
	{
		PROPVARIANT pvSysMode;
		PropVariantInit(&pvSysMode);
		pvSysMode.vt = VT_I4;
		//pvSysMode.lVal = micspk->opt & AMS2_OPT_AEC ? SINGLE_CHANNEL_AEC : SINGLE_CHANNEL_NSAGC;
		pvSysMode.lVal = micspk->opt & AMS2_OPT_AEC ? OPTIBEAM_ARRAY_AND_AEC : SINGLE_CHANNEL_NSAGC;
		pPS->lpVtbl->SetValue(pPS, &MFPKEY_WMAAECMA_SYSTEM_MODE, &pvSysMode);
		PropVariantClear(&pvSysMode);
	}

#if 1
	// feature mode
	{
		PROPVARIANT pvFeatMode;
		PropVariantInit(&pvFeatMode);
		pvFeatMode.vt = VT_BOOL;
		pvFeatMode.boolVal = VARIANT_TRUE;		
		pPS->lpVtbl->SetValue(pPS, &MFPKEY_WMAAECMA_FEATURE_MODE, &pvFeatMode);
		PropVariantClear(&pvFeatMode);
	}
#endif

	// set device index
	{
		PROPVARIANT pvDeviceId;
		PropVariantInit(&pvDeviceId);
		pvDeviceId.vt = VT_I4;
		pvDeviceId.lVal = (unsigned long)(micspk->ply_dev_id<<16) + (unsigned long)(0x0000ffff & micspk->rec_dev_id);
		pPS->lpVtbl->SetValue(pPS, &MFPKEY_WMAAECMA_DEVICE_INDEXES, &pvDeviceId);
		PropVariantClear(&pvDeviceId);
	}

	// echo length
#if 0
	if (micspk->opt & AMS2_OPT_AEC)
	{
		PROPVARIANT pvEchoLength;
		PropVariantInit(&pvEchoLength);
		pvEchoLength.vt = VT_I4;
		pvEchoLength.lVal = 128;
		pPS->lpVtbl->SetValue(pPS, &MFPKEY_WMAAECMA_FEATR_ECHO_LENGTH, &pvEchoLength);
		PropVariantClear(&pvEchoLength);
	}
#endif

	// echo supress times
#if 1
	if (micspk->opt & AMS2_OPT_AEC)
	{
		PROPVARIANT pvAESTimes;
		PropVariantInit(&pvAESTimes);
		pvAESTimes.vt = VT_I4;
		pvAESTimes.lVal = 1; // 0, 1, 2
		pPS->lpVtbl->SetValue(pPS, &MFPKEY_WMAAECMA_FEATR_AES, &pvAESTimes);
		PropVariantClear(&pvAESTimes);
	}
#endif

	// center clip
#if 0
	if (micspk->opt & AMS2_OPT_AEC) 
	{
		PROPVARIANT pvCenterClip;
		PropVariantInit(&pvCenterClip);
		pvCenterClip.vt = VT_BOOL;
		pvCenterClip.boolVal = VARIANT_TRUE;		
		pPS->lpVtbl->SetValue(pPS, &MFPKEY_WMAAECMA_FEATR_CENTER_CLIP, &pvCenterClip);
		PropVariantClear(&pvCenterClip);
	}
#endif

#if 1

	// noise fill
	if (micspk->opt & AMS2_OPT_AEC)
	{
		PROPVARIANT pvNSFill;
		PropVariantInit(&pvNSFill);
		pvNSFill.vt = VT_BOOL;
		pvNSFill.boolVal = VARIANT_TRUE;		
		pPS->lpVtbl->SetValue(pPS, &MFPKEY_WMAAECMA_FEATR_NOISE_FILL, &pvNSFill);
		PropVariantClear(&pvNSFill);
	}

	// retreive TS
	if (micspk->opt & AMS2_OPT_AEC) 
	{
		PROPVARIANT pvTS;
		PropVariantInit(&pvTS);
		pvTS.vt = VT_BOOL;
		pvTS.boolVal = VARIANT_TRUE;		
		pPS->lpVtbl->SetValue(pPS, &MFPKEY_WMAAECMA_RETRIEVE_TS_STATS, &pvTS);
		PropVariantClear(&pvTS);
	}
#endif

	// noise suppression
	{
		PROPVARIANT pvNoiseSup;
		PropVariantInit(&pvNoiseSup);
		pvNoiseSup.vt = VT_I4;
		pvNoiseSup.lVal = micspk->opt & AMS2_OPT_NS;
		pPS->lpVtbl->SetValue(pPS, &MFPKEY_WMAAECMA_FEATR_NS, &pvNoiseSup);
		PropVariantClear(&pvNoiseSup);
	}

#if 1
	// Turn on/off AGC
	{
		PROPVARIANT pvAGC;
		PropVariantInit(&pvAGC);
		pvAGC.vt = VT_BOOL;
		pvAGC.boolVal = micspk->opt & AMS2_OPT_AGC ? VARIANT_TRUE : VARIANT_FALSE;
		pPS->lpVtbl->SetValue(pPS, &MFPKEY_WMAAECMA_FEATR_AGC, &pvAGC);
		PropVariantClear(&pvAGC);
	}
#endif

	// set output format
	{
		DMO_MEDIA_TYPE mt = {0};
		WAVEFORMATEX wfxOut = {WAVE_FORMAT_PCM, 1, CAPTURE_AUDIO_CLOCK, CAPTURE_AUDIO_CLOCK<<1, 2, 16, 0};
		MoInitMediaType(&mt, sizeof(WAVEFORMATEX));
		mt.majortype = MEDIATYPE_Audio;
		mt.subtype = MEDIASUBTYPE_PCM;
		mt.lSampleSize = 0;
		//mt.lSampleSize = 2;
		mt.bFixedSizeSamples = TRUE;
		mt.bTemporalCompression = FALSE;
		mt.formattype = FORMAT_WaveFormatEx;
		memcpy(mt.pbFormat, &wfxOut, sizeof(WAVEFORMATEX));
		pDMO->lpVtbl->SetOutputType(pDMO, 0, &mt, 0); 
		micspk->rec_buf_size = wfxOut.nSamplesPerSec * wfxOut.wBitsPerSample / 8;
		MoFreeMediaType(&mt);
	}

	pDMO->lpVtbl->AllocateStreamingResources(pDMO);

	pPS->lpVtbl->Release(pPS);

	micspk->rec_dmo = pDMO;

	return 0;
}

static void dump_aec_config(audiomicspk2_t *micspk)
{
	IMediaObject *pDMO = (IMediaObject*)micspk->rec_dmo;
	IPropertyStore *pPS = NULL;
	HRESULT hr;
	int echo_length = 0;
	int frame_size = 0;
	int aes_times = 0;

	hr = pDMO->lpVtbl->QueryInterface(pDMO, &IID_IPropertyStore, (void**)&pPS);

	//if (micspk->opt & AMS2_OPT_AEC)
	{
		PROPVARIANT pvEchoLength;
		PropVariantInit(&pvEchoLength);
		pvEchoLength.vt = VT_I4;
		//pvEchoLength.lVal = 128;
		pPS->lpVtbl->GetValue(pPS, &MFPKEY_WMAAECMA_FEATR_ECHO_LENGTH, &pvEchoLength);
		echo_length = pvEchoLength.lVal;
		PropVariantClear(&pvEchoLength);
	}

	//if (micspk->opt & AMS2_OPT_AEC)
	{
		PROPVARIANT pvFrameSize;
		PropVariantInit(&pvFrameSize);
		pvFrameSize.vt = VT_I4;
		//pvEchoLength.lVal = 128;
		pPS->lpVtbl->GetValue(pPS, &MFPKEY_WMAAECMA_FEATR_FRAME_SIZE, &pvFrameSize);
		frame_size = pvFrameSize.lVal;
		PropVariantClear(&pvFrameSize);
	}

	//if (micspk->opt & AMS2_OPT_AEC)
	{
		PROPVARIANT pvAESTimes;
		PropVariantInit(&pvAESTimes);
		pvAESTimes.vt = VT_I4;
		//pvAESTimes.lVal = 1; // 0, 1, 2
		pPS->lpVtbl->GetValue(pPS, &MFPKEY_WMAAECMA_FEATR_AES, &pvAESTimes);
		aes_times = pvAESTimes.lVal;
		PropVariantClear(&pvAESTimes);
	}

	if (0)
	{
		AecQualityMetrics_Struct *pAECQuality;
		PROPVARIANT pvAECQuality;
		PropVariantInit(&pvAECQuality);
		hr = pPS->lpVtbl->GetValue(pPS, &MFPKEY_WMAAECMA_QUALITY_METRICS, &pvAECQuality);
		if (SUCCEEDED(hr)) {
			pAECQuality = (AecQualityMetrics_Struct *)pvAECQuality.blob.pBlobData;
			AUDIO_LOG_INFO("i64Timestamp:%lld, ConvergenceFlag:%d, MicClippedFlag:%d, MicSilenceFlag:%d,"
				"PstvFeadbackFlag:%d, SpkClippedFlag:%d, SpkMuteFlag:%d, GlitchFlag:%d, DoubleTalkFlag:%d,"
				"uGlitchCount:%d,uMicClipCount:%d,fDuration:%f,fTSVariance:%f,fTSDriftRate:%f"
				"fVoiceLevel:%f,fNoiseLevel:%f,fERLE:%f,fAvgERLE:%f,dwReserved:%d",
				pAECQuality->i64Timestamp, pAECQuality->ConvergenceFlag, pAECQuality->MicClippedFlag, 
				pAECQuality->MicSilenceFlag, pAECQuality->PstvFeadbackFlag, pAECQuality->SpkClippedFlag, 
				pAECQuality->SpkMuteFlag, pAECQuality->GlitchFlag, pAECQuality->DoubleTalkFlag, 
				pAECQuality->uGlitchCount, pAECQuality->uMicClipCount, pAECQuality->fDuration, pAECQuality->fTSVariance,
				pAECQuality->fTSDriftRate, pAECQuality->fVoiceLevel, pAECQuality->fNoiseLevel, pAECQuality->fERLE, 
				pAECQuality->fAvgERLE, pAECQuality->dwReserved);
			if (!pAECQuality->ConvergenceFlag) {
				pAECQuality->ConvergenceFlag = 1;
				pAECQuality->uMicClipCount = 4;
				pAECQuality->fERLE = 0.009828;
				pAECQuality->fAvgERLE = 0.172231;
				hr = pPS->lpVtbl->SetValue(pPS, &MFPKEY_WMAAECMA_QUALITY_METRICS, &pvAECQuality);
				if (SUCCEEDED(hr)) {
					AUDIO_LOG_INFO("set MFPKEY_WMAAECMA_QUALITY_METRICS ok!");
				} else {
					AUDIO_LOG_INFO("set MFPKEY_WMAAECMA_QUALITY_METRICS failed!");
				}
			}
#if 0
			if (pAECQuality->ConvergenceFlag) {
				hr = pPS->lpVtbl->SetValue(pPS, &MFPKEY_WMAAECMA_QUALITY_METRICS, &pvAECQuality);
				if (SUCCEEDED(hr)) {
					AUDIO_LOG_INFO("set MFPKEY_WMAAECMA_QUALITY_METRICS ok!");
				} else {
					AUDIO_LOG_INFO("set MFPKEY_WMAAECMA_QUALITY_METRICS failed!");
				}
			}
#endif
		}
		PropVariantClear(&pvAECQuality);
	}

	AUDIO_LOG_INFO("echo_length:%d, frame_size:%d, aes_times:%d", echo_length, frame_size, aes_times);

	if (pPS) {
		pPS->lpVtbl->Release(pPS);
	}
}

static void set_aec_config(audiomicspk2_t *micspk)
{
	IMediaObject *pDMO = (IMediaObject*)micspk->rec_dmo;
	IPropertyStore *pPS = NULL;
	HRESULT hr;
	int echo_length = 0;
	int frame_size = 0;

	hr = pDMO->lpVtbl->QueryInterface(pDMO, &IID_IPropertyStore, (void**)&pPS);

	// config AEC runtime state
	{
		AecQualityMetrics_Struct AECQuality = {0};
		PROPVARIANT pvAECQuality;
		PropVariantInit(&pvAECQuality);
		pvAECQuality.vt = VT_BLOB;
		pvAECQuality.blob.cbSize = sizeof(AECQuality);
		pvAECQuality.blob.pBlobData = &AECQuality;
		AECQuality.ConvergenceFlag = 1;
		AECQuality.uMicClipCount = 371;
		AECQuality.fDuration = 134.690002;
		AECQuality.fTSVariance = 0.167386;
		AECQuality.fTSDriftRate = 0.000002;
		AECQuality.fVoiceLevel  =0.106498;
		AECQuality.fNoiseLevel = 0.004691;
		AECQuality.fERLE = 6.605908;
		AECQuality.fAvgERLE = 3.306793;
		hr  = pPS->lpVtbl->SetValue(pPS, &MFPKEY_WMAAECMA_QUALITY_METRICS, &pvAECQuality);
		PropVariantClear(&pvAECQuality);
		if (SUCCEEDED(hr)) {
			AUDIO_LOG_INFO("set MFPKEY_WMAAECMA_QUALITY_METRICS ok!");
		}
	}

	if (pPS) {
		pPS->lpVtbl->Release(pPS);
	}
}

static void uninitialize_spk(audiomicspk2_t *micspk)
{
	if (micspk->ply_stream) {
		Pa_AbortStream(micspk->ply_stream);
		Pa_CloseStream(micspk->ply_stream);
	}

	if (micspk->ply_dbuf) {
		delay_buf_destroy(micspk->ply_dbuf);
		micspk->ply_dbuf = NULL;
	}
}

static void uninitialize_rec(audiomicspk2_t *micspk)
{
	IMediaObject *pDMO = (IMediaObject *)micspk->rec_dmo;

	if (pDMO) {
		pDMO->lpVtbl->Flush(pDMO);
		pDMO->lpVtbl->FreeStreamingResources(pDMO);
		pDMO->lpVtbl->Release(pDMO);
	}
	
	if (micspk->rec_dbuf) {
		delay_buf_destroy(micspk->rec_dbuf);
		micspk->rec_dbuf = NULL;
	}
	
	micspk->rec_dmo = NULL;
}

static void on_clock_rec(audiomicspk2_t *micspk)
{
	IMediaObject *pDMO = (IMediaObject*)micspk->rec_dmo;
	DWORD dwStatus = 0;
	HRESULT hr;
	DWORD cOutputBufLen = micspk->rec_buf_size;
	DMO_OUTPUT_DATA_BUFFER bufferStruct = {0};

	do {
		bufferStruct.pBuffer = create_media_buffer(cOutputBufLen);
		bufferStruct.dwStatus = 0;

		hr = pDMO->lpVtbl->ProcessOutput(pDMO, 0, 1, &bufferStruct, &dwStatus);

		if (SUCCEEDED(hr)) {
			if (hr == S_FALSE) {
				//....
			} else {
				save_data(micspk, bufferStruct.pBuffer);
#if 0
				if (micspk->opt & AMS2_OPT_AEC) {
					dump_aec_config(micspk);
				}
#endif
			}

			bufferStruct.pBuffer->lpVtbl->Release(bufferStruct.pBuffer);
		} else {
			bufferStruct.pBuffer->lpVtbl->Release(bufferStruct.pBuffer);
			break;
		}

	} while (bufferStruct.dwStatus & DMO_OUTPUT_DATA_BUFFERF_INCOMPLETE);
	
}

static unsigned int __stdcall work_proc(void *arg)
{
	audiomicspk2_t *micspk = (audiomicspk2_t *)arg;
	int rc;

	// set a higher priority because of audio is very sensitive about timing
	SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_HIGHEST);

	CoInitialize(NULL);

	//
	// record need play because of AEC, so 
	// record <---> record and play
	// play   <---> play
	// record and play <---> record and play
	//
	if (micspk->opt & AMS_OPT_RECORD) { 
		rc = initialize_spk(micspk);
		if (rc != 0){
			AUDIO_LOG_ERROR("record mode initialize speak louder param failed!");
			goto on_error;
		}
		Sleep(100); // play before record
		rc = initialize_rec(micspk);
		if (rc != 0){
			AUDIO_LOG_ERROR("initialize micro capture param failed!");
			goto on_error;
		}
	} else if (micspk->opt & AMS_OPT_PLAY) {
		rc = initialize_spk(micspk);
		if (rc != 0){
			AUDIO_LOG_ERROR("play mode initialize speak louder param failed!");
			goto on_error;
		}
	}

	micspk->evt_exit = CreateEventA(NULL, FALSE, FALSE, NULL);

	for (;;) {
		DWORD dwRet = WaitForSingleObject(micspk->evt_exit, CLOCK_PERIOD);
		if (dwRet == WAIT_OBJECT_0) { // exit
			break;
		} else { // timeout
			if (micspk->opt & AMS_OPT_RECORD) {
				on_clock_rec(micspk);
			} else if (micspk->opt & AMS_OPT_PLAY) {
				//on_clock_spk(micspk);
			}
		}
	}

on_error:

	if (micspk->opt & AMS_OPT_RECORD) {
		uninitialize_rec(micspk);
		uninitialize_spk(micspk);
	} else if (micspk->opt & AMS_OPT_PLAY) {
		uninitialize_spk(micspk);
	}

	CoUninitialize();

	return 0;
}

static apr_status_t read_frame(void *self, audioframe_t *frame)
{
	audiomicspk2_t *micspk = CONTAINING_RECORD(self, audiomicspk2_t, base);
	UINT32 frame_samples = AUDIO_CLOCK * FRAME_TIME / 1000;

	frame->size = 2*frame_samples;
	frame->dtmf = 0;

	delay_buf_get(micspk->rec_dbuf, (short*)frame->buffer);

	return APR_SUCCESS;	
}

static apr_status_t write_frame(void *self, const audioframe_t *frame)
{
	audiomicspk2_t *micspk = CONTAINING_RECORD(self, audiomicspk2_t, base);

	delay_buf_put(micspk->ply_dbuf, (short*)frame->buffer);

	return APR_SUCCESS;
}

static audiostream_vtbl_t g_stream_vtbl = {
	&read_frame,
	&write_frame,
};

apr_status_t audiomicspk2_create(apr_pool_t *pool,
	audioengine_t *engine, 
	int opt, 
	int clock,
	const char *rec_dev_key,
	const char *ply_dev_key,
	audiomicspk2_t **p_micspk)
{
	int rec_dev_id = -1; 
	int ply_dev_id = -1;
	int ply_dev_id_portaudio = -1;
	audiomicspk2_t *micspk;
	unsigned long frame_samples;
	unsigned long capture_frame_samples;

	ply_dev_id_portaudio = get_device_index_poartaudio(0, ply_dev_key);
	ply_dev_id = get_device_index(0, ply_dev_key);
	rec_dev_id = get_device_index(1, rec_dev_key);

	if (rec_dev_id < 0 || ply_dev_id < 0 || ply_dev_id_portaudio < 0)
		return APR_EGENERAL;

	//opt |= AMS2_OPT_AGC | AMS2_OPT_NS;

	micspk = apr_palloc(pool, sizeof(audiomicspk2_t));
	memset(micspk, 0, sizeof(audiomicspk2_t));

	frame_samples = FRAME_TIME * clock / 1000;
	capture_frame_samples = FRAME_TIME * CAPTURE_AUDIO_CLOCK/ 1000;

	micspk->rec_dev_id = rec_dev_id;
	micspk->ply_dev_id = ply_dev_id;
	micspk->ply_dev_id_portaudio = ply_dev_id_portaudio;
	micspk->opt = opt;
	micspk->frame_samples = frame_samples;
	micspk->capture_frame_samples = capture_frame_samples;

	audiostream_init(engine, &g_stream_vtbl, &micspk->base);
	micspk->base.direction = 0;

	if (opt & AMS_OPT_PLAY) {
		micspk->base.direction |= STREAM_DIR_WRITE;
		delay_buf_create(clock, frame_samples, 1, MAX_DELAY, 0, (delay_buf**)&micspk->ply_dbuf);
		micspk->ply_buf = (short*)apr_palloc(pool, frame_samples<<1);
		micspk->ply_buf_cnt = 0;
		micspk->output_resample_state = speex_resampler_init(1, AUDIO_CLOCK, CAPTURE_AUDIO_CLOCK, RESAMPLE_QUALITY, NULL);
	}
	if (opt & AMS_OPT_RECORD) {
		micspk->base.direction |= STREAM_DIR_READ;
		delay_buf_create(clock, frame_samples, 1, MAX_DELAY, 0, (delay_buf**)&micspk->rec_dbuf);
		micspk->input_resample_state = speex_resampler_init(1, CAPTURE_AUDIO_CLOCK, AUDIO_CLOCK, RESAMPLE_QUALITY, NULL);
	}

	micspk->worker_thread = (HANDLE)_beginthreadex(NULL, 0, &work_proc, micspk, 0, NULL);

	if (micspk->worker_thread) {
		BOOL bThreadExited = FALSE;
		do {
			DWORD dwRet = WaitForSingleObject(micspk->worker_thread, 10);
			if (dwRet == WAIT_TIMEOUT) {
				volatile HANDLE hExit = micspk->evt_exit;
				if (hExit)
					break;
			} else if (dwRet == WAIT_OBJECT_0) {
				bThreadExited = TRUE;
			}
		} while (!bThreadExited);
		if (bThreadExited) {
			audiomicspk2_destroy(micspk);
			return APR_EGENERAL;
		}
	} else {
		audiomicspk2_destroy(micspk);
		return APR_EGENERAL;
	}

	*p_micspk = micspk;

	return APR_SUCCESS;
}

void audiomicspk2_destroy(audiomicspk2_t *micspk)
{
	if (micspk->evt_exit) {
		SetEvent(micspk->evt_exit);
		WaitForSingleObject(micspk->worker_thread, INFINITE);
		CloseHandle(micspk->worker_thread);
		CloseHandle(micspk->evt_exit);
	}
	if (micspk->input_resample_state) {
		speex_resampler_destroy((SpeexResamplerState*)micspk->input_resample_state);
		micspk->input_resample_state = NULL;
	}
	if (micspk->output_resample_state) {
		speex_resampler_destroy((SpeexResamplerState*)micspk->output_resample_state);
		micspk->output_resample_state = NULL;
	}
}