myvtm/Module/mod_sipphone/audio_session.cpp

#include "stdafx.h"
#include "SpBase.h"
#include "audio_session.h"
#include "Event.h"


#include <memutil.h>
#include <rtp.h>
#include <rtpsession.h>

#include "../../Other/libaudioframework/audioframework.h"

#include "../../Other/libaudioqueue/libaudioqueue.h"

#include "../../Other/rvcmediacommon/rvc_media_common.h"
#include <string.h>

#include "../../Other/libaudions/iaudionsinterface.h"

#ifdef RVC_OS_WIN
#include <portaudio.h>
#endif

#define AUDIO_CLOCK	8000

#define AUDIO_SHM_FRAME_TIME	20 // 20ms

#ifndef RVC_MAX_BUFFER_LEN
#define RVC_MAX_BUFFER_LEN 1024
#endif

#ifndef RVC_AUDIO_FRAME_LEN
#define RVC_AUDIO_FRAME_LEN	320
#endif

char straudiodata[RVC_MAX_BUFFER_LEN] = {0};
int iIndex = 0;
int iLastLeft = 0;

static int g_nAudioRecvNum = 0;
static int g_nAudioSendNum = 0;

enum e_media_dir 
{
	DIR_NONE = 0,
	DIR_TX = 1, 
	DIR_RX = 2,
	DIR_BOTH = 3,
};

typedef struct audio_recorder_t audio_recorder_t;
typedef struct audio_phonemedia_t audio_phonemedia_t;

struct audio_session_t
{
	audio_session_conf_t conf;

	audio_session_phonemedia_conf_t phonemedia_conf;
	audio_session_t *owner;

	apr_pool_t *pool;
	audioengine_t *engine;
	audiocontext_t *context;

	audiobridge_t *bridge;
	apr_pool_t *micspk_pool;

#ifdef _WIN32
	audiomicspk2_t* micspkstream;
#else
	audiomicspkpulse_t* micspkstream;
#endif // _WIN32

	audiodsp_t *dspstream;
	audioresize_t *resizestream;
	audiortp_t *rtpstream;
	audiocodec_t *codecstream;
	rtp_session_t *rtpsess;

	struct 
	{
		audioresize_t *resizestream;
		audiortp_t *rtpstream;
		audiocodec_t *codecstream;
		rtp_session_t *rtpsess;
		int state;
		audio_session_remote_recording_conf_t conf;
	}record;
	Clibaudioqueue* remoteaudioqueue;
	bool brtpinsertqueue;
	//audio noise suppression
	IAudioNs* audionsobj;
	IAudioNs* audioplaynsobj;
	bool baudiorecved;
};

static int rx_audio_callback(char *frame,void*userdata)
{
	audio_session_t*session = (audio_session_t*)userdata;
	int used = 0;

	if (DOUBLERECORD_CALLTYPE != session->phonemedia_conf.eCalltype)
	{
		if (frame) 
		{
			audio_frame frm;
			frm.bitspersample = 16;
			frm.format = 1;
			frm.data = frame;
			frm.framesize = 160;   //注意此参数可能不准确，网络传输的包大小可能是不定长的，取音频数据时慎用此参数
			//写入实际的单个包大小
			//frm.framesize = strlen(frame);  //不能使用此方法，网络传输的包大小可能是不定长的
			frm.nchannels = 1;
			frm.samplespersec = 8000;
			frm.iseriesnumber = g_nAudioRecvNum;
			if (!session->remoteaudioqueue->InsertAudio(&frm)) 
			{
				Dbg("InsertAudio failed! frameCount:%d", frm.framesize);
				used = -1;
			}
		}
	}
	return used;
}


static int rvc_audio_ns(void* pdst, size_t udstlen, void* psrc, size_t usrclen, void* user_data)
{
	int iret = -1;
	audio_session_t* session = (audio_session_t*)user_data;
	if (NULL != pdst && NULL != psrc){
		if (NULL != session && NULL != session->audionsobj){
			session->audionsobj->NsProcess((char*)pdst, udstlen, (char*)psrc, usrclen);
			//memcpy(pdst, psrc, usrclen);
			iret = 0;
		}
	}

	return iret;
}


static int rvc_audio_play_ns(void* pdst, size_t udstlen, void* psrc, size_t usrclen, void* user_data)
{
	int iret = -1;
	audio_session_t* session = (audio_session_t*)user_data;
	if (NULL != pdst && NULL != psrc) {
		if (NULL != session->audioplaynsobj) {
			session->audioplaynsobj->NsProcess((char*)pdst, udstlen, (char*)psrc, usrclen);
			//memcpy(pdst, psrc, usrclen);
			iret = 0;
		}
	}

	return iret;
}

static int tx_audio_callback(void* audiodata, void* userdata)
{
	audio_session_t* session = (audio_session_t*)userdata;
	int used = 0;

	return used;
}

static void send_hook_callback(const char *buf, int size, void *arg)
{
	//LOG_FUNCTION();
	//Dbg("send audio pkt");
	if ((g_nAudioSendNum%60) ==0)
	{
		//Dbg("send audio pkt num %d,single size %d",g_nAudioSendNum,size);
	}
	g_nAudioSendNum++;
}

static bool phonemedia_rtp_record(audio_session_t* pseesion)
{
	bool bret = false;
	if (NULL == pseesion){
		return bret;
	}

	if (true == pseesion->brtpinsertqueue){
		if (DOUBLERECORD_CALLTYPE == pseesion->phonemedia_conf.eCalltype){
			if (eStand2sType == pseesion->phonemedia_conf.eDeviceType){
				if (DEV_PICKUP == pseesion->phonemedia_conf.dev_type){
					bret = true;
				}
			}
		}
	}

	return bret;
}

static void recv_hook_callback(const char *buf, int size, void *arg)
{
	//LOG_FUNCTION();
	rtp_hdr *hdr = (rtp_hdr*)buf;
	audio_session_t* psession = (audio_session_t*)arg;

	if (false == psession->baudiorecved){
		char strmsg[MAX_PATH] = { 0 };
		snprintf(strmsg, MAX_PATH, "received first audio packet, and packet size is %d.", size);
		LogWarn(Severity_Middle, Error_Debug, EVENT_MOD_SIP_AUDIO_STREAM_RECEIVED, strmsg);
		psession->baudiorecved = true;
	}

	if ((g_nAudioRecvNum%100) == 0)
	{
		//Dbg("recv audio pkt num %d,single size %d",g_nAudioRecvNum,size);
		static int icount = 0;
		if (psession->phonemedia_conf.eDeviceType == eStand2sType && icount == 0){
			icount++;
			Dbg("current hand free flag is %d, call type = %d, pt = %d,arg addr is 0x%08x.",(int)psession->phonemedia_conf.dev_type, psession->phonemedia_conf.eCalltype, hdr->pt, arg);
		}
	}
	
	//if (0 == pseesion->phonemedia_conf.dev_type && DOUBLERECORD_CALLTYPE == pseesion->phonemedia_conf.eCalltype){
	//	if (false == pseesion->brtpinsertqueue){
	//		Dbg("rtp stream insert to audio queue flag is set to true.");
	//	}
	//	pseesion->brtpinsertqueue = true;
	//}
	g_nAudioRecvNum++;

	//if (true == phonemedia_rtp_record(pseesion)){
	//	char strbuffer[RVC_MAX_BUFFER_LEN]={0};
	//	int outsize = RVC_MAX_BUFFER_LEN;
	//	switch(hdr->pt)
	//	{
	//	case RTP_PT_PCMA:
	//		audiocodec_pcma_decode(buf+sizeof(rtp_hdr), size-sizeof(rtp_hdr), strbuffer, &outsize);
	//		break;
	//	case RTP_PT_PCMU:
	//		audiocodec_pcmu_decode(buf+sizeof(rtp_hdr), size-sizeof(rtp_hdr), strbuffer, &outsize);
	//		break;
	//	case RTP_PT_G729:
	//		audiocodec_g729a_decode(buf+sizeof(rtp_hdr), size-sizeof(rtp_hdr), strbuffer, &outsize);
	//		break;
	//	default:
	//		Dbg("audiocodec_decode not support audio pt(%d).", hdr->pt);
	//		break;
	//	}

	//	if (pseesion && pseesion->remoteaudioqueue){
	//		int iCount = (outsize+iLastLeft)/RVC_AUDIO_FRAME_LEN;
	//		memcpy(straudiodata+iLastLeft, strbuffer, iCount*RVC_AUDIO_FRAME_LEN-iLastLeft);
	//		for(int i = 0; i < iCount; i++)
	//		{
	//			audio_frame frm;
	//			char straudio[RVC_AUDIO_FRAME_LEN]={0};
	//			memcpy(straudio, straudiodata+i*RVC_AUDIO_FRAME_LEN, RVC_AUDIO_FRAME_LEN);
	//			frm.bitspersample = 16;
	//			frm.format = 1;
	//			frm.data = straudio;
	//			frm.framesize = RVC_AUDIO_FRAME_LEN;   //注意此参数可能不准确，网络传输的包大小可能是不定长的，取音频数据时慎用此参数
	//			frm.nchannels = 1;
	//			frm.samplespersec = 8000;
	//			if (!pseesion->remoteaudioqueue->InsertAudio(&frm)) 
	//			{
	//				Dbg("InsertAudio failed! frameCount:%d", frm.framesize);
	//			}
	//		}
	//		memset(straudiodata, 0, RVC_MAX_BUFFER_LEN);									//清空缓存
	//		iLastLeft = (outsize + iLastLeft) % RVC_AUDIO_FRAME_LEN;						//上次剩余不足RVC_AUDIO_FRAME_LEN的buffer
	//		if ((0 != iLastLeft) && (iCount*RVC_AUDIO_FRAME_LEN < outsize)){
	//			memcpy(straudiodata, strbuffer+iCount*RVC_AUDIO_FRAME_LEN, iLastLeft);		//暂存上一次的未入队列的音频数据
	//		}
	//	}
	//	else{
	//		Dbg("pseesion->remoteaudioqueue is null.");
	//	}
	//}
}


static void audio_device_event(bool bopen, int iret, bool bmicro, int idev, const char* strmessage, void* user_data)
{
	char strinfo[MAX_PATH] = { 0 };
	DWORD errorcode = 0;
	if (bopen){
		if (DEV_PICKUP == idev) {
			char strpickup[] = "[pickup]";
			snprintf(strinfo, MAX_PATH, "%s%s", strmessage, strpickup);
			if (0 == iret) {
				if (bmicro) {
					errorcode = EVENT_MOD_SIP_PICKUP_IN_AUDIO_DEVICE_OPEN_SUCCESS;
				}
				else {
					errorcode = EVENT_MOD_SIP_PICKUP_OUT_AUDIO_DEVICE_OPEN_SUCCESS;
				}
			}
			else {
				if (bmicro) {
					errorcode = EVENT_MOD_SIP_PICKUP_IN_AUDIO_DEVICE_OPEN_FAILED;
				}
				else {
					errorcode = EVENT_MOD_SIP_PICKUP_OUT_AUDIO_DEVICE_OPEN_FAILED;
				}
			}
		}
		else {
			char strhandfree[] = "[hand free]";
			snprintf(strinfo, MAX_PATH, "%s%s", strmessage, strhandfree);
			if (0 == iret) {
				if (bmicro) {
					errorcode = EVENT_MOD_SIP_HANDFREE_IN_AUDIO_DEVICE_OPEN_SUCCESS;
				}
				else {
					errorcode = EVENT_MOD_SIP_HANDFREE_OUT_AUDIO_DEVICE_OPEN_SUCCESS;
				}
			}
			else {
				if (bmicro) {
					errorcode = EVENT_MOD_SIP_HANDFREE_IN_AUDIO_DEVICE_OPEN_FAILED;
				}
				else {
					errorcode = EVENT_MOD_SIP_HANDFREE_OUT_AUDIO_DEVICE_OPEN_FAILED;
				}
			}
		}
	}
	else {
		if (DEV_PICKUP == idev) {
			char strpickup[] = "[pickup]";
			snprintf(strinfo, MAX_PATH, "%s%s", strmessage, strpickup);
			if (bmicro) {
				errorcode = EVENT_MOD_SIP_PICKUP_IN_AUDIO_DEVICE_CLOSE;
			}
			else {
				errorcode = EVENT_MOD_SIP_PICKUP_OUT_AUDIO_DEVICE_CLOSE;
			}
		}
		else {
			char strhandfree[] = "[hand free]";
			snprintf(strinfo, MAX_PATH, "%s%s", strmessage, strhandfree);
			if (bmicro) {
				errorcode = EVENT_MOD_SIP_HANDFREE_IN_AUDIO_DEVICE_CLOSE;
			}
			else {
				errorcode = EVENT_MOD_SIP_HANDFREE_OUT_AUDIO_DEVICE_CLOSE;
			}
		}
	}

	LogWarn(Severity_Middle, Error_Debug, errorcode, strinfo);
}

//static int translate_id(int in_direction, int idx);

static int phonemedia_stop(audio_session_t *session, int b_record_turn_off);

static void phonemedia_reconfig(audio_session_t *media, const audio_session_phonemedia_conf_t *conf)
{
	memcpy(&media->phonemedia_conf, conf, sizeof(audio_session_phonemedia_conf_t));
}

#if defined(RVC_OS_LINUX)
static int execute_cmd_return_result(const char* cmd, char* result)
{
    char buf_ps[1024];
    char ps[1024] = { 0 };
    FILE* ptr;
    strcpy(ps, cmd);
    if ((ptr = popen(ps, "r")) != NULL) {
        while (fgets(buf_ps, 1024, ptr) != NULL) {
            strcat(result, buf_ps);
            if (strlen(result) > 1024)
                break;
        }
        pclose(ptr);
        return 0;
    } else {
		sprintf(result, "popen %s error: %d", ps, errno);
        return -1;
    }
}

#endif //RVC_OS_LINUX

static int phonemedia_on_remote_recording(audio_session_t *session)
{
	int rc = 0;
	audio_session_t *media = session;
	assert(media->record.state);

	audiocontext_remove_driver(media->context, &media->bridge->base);
	{
		audio_session_remote_recording_conf_t *conf = &media->record.conf;
		int clock = REC_COMMON_AUDIO_CLOCK;
		int ptime = REC_COMMON_AUDIO_FRAME_PTIME;
		apr_status_t status;
		rc = rtp_session_create2(conf->local_rtp_ip, conf->local_rtp_port, 2, &media->record.rtpsess);
		if (rc != 0) {
			char msg[256];
			const int lastErr = WSAGetLastError();
			sprintf(msg, "create rtp session failed! rtp:%d, WSALastError=%d, rc=%d", conf->local_rtp_port, lastErr, rc);
			Dbg(msg);
#if defined(RVC_OS_LINUX)
            LogWarn(Severity_Low, Error_Unexpect, ERROR_MOD_SIP_SES_CREATE_FAILED, msg);
            if (lastErr == 10048) { //EADDRINUSE
                char content[1025];
                char cmd[256];
				sprintf(cmd, "netstat -tunlp | grep -E '%d|%d'", conf->local_rtp_port, conf->local_rtp_port+1);
				execute_cmd_return_result(cmd, content);
				sprintf(cmd, " | cur pid: %d", getpid());
				strcat(content, cmd);
				LogWarn(Severity_Low, Error_Unexpect, ERROR_MOD_SIP_SES_SOCKET_INUSED, content);
            }
#endif //RVC_OS_LINUX


			goto on_error;
		}
		rtp_session_reset2(media->record.rtpsess, RTP_SESSION_FLAG_SENDONLY|RTP_SESSION_FLAG_NO_RTCP, conf->remote_rtp_ip, conf->remote_rtp_port, conf->remote_rtp_port+1);
		status = audioresize_create(media->pool, media->engine, FRAME_TIME*2*clock/1000, ptime*2*clock/1000, FRAME_TIME*2*clock/1000, ptime*2*clock/1000, &media->record.resizestream);
		if (status != APR_SUCCESS)
			goto on_error;
		status = audiocodec_create(media->pool, media->engine, "G729", clock, FRAME_TIME, AUDIO_CODEC_OPT_ENCODE_WRITE, &media->record.codecstream);
		if (status != APR_SUCCESS)
			goto on_error;
		status = audiortp_create(media->pool, media->engine, media->record.rtpsess, &media->record.rtpstream);
		if (status != APR_SUCCESS)
			goto on_error;
		{
			int param;
			param = clock;
			audiortp_set_param(media->record.rtpstream, AUDIO_RTP_FLAG_SEND_CLOCK, &param);
			audiortp_set_param(media->record.rtpstream, AUDIO_RTP_FLAG_RECV_CLOCK, &param);
			param = REC_COMMON_AUDIO_PT;
			audiortp_set_param(media->record.rtpstream, AUDIO_RTP_FLAG_SEND_PT, &param);
			audiortp_set_param(media->record.rtpstream, AUDIO_RTP_FLAG_RECV_PT, &param);
			param = ptime;
			audiortp_set_param(media->record.rtpstream, AUDIO_RTP_FLAG_SEND_PTIME, &param);
			audiortp_set_param(media->record.rtpstream, AUDIO_RTP_FLAG_RECV_PTIME, &param);
			//audiortp_set_param(media->record.rtpstream, AUDIO_RTP_FLAG_HOOK_ARG, media);
			audiortp_init(media->record.rtpstream);
		}
		audiostream_connect_pipeline(STREAM_DIR_WRITE, &media->record.resizestream->base, &media->record.codecstream->base, &media->record.rtpstream->base, NULL);
		audiobridge_set_recorder(media->bridge, &media->record.resizestream->base);

	on_error:

		if (status != APR_SUCCESS) {
			Dbg("create remote recording objects failed!");
			rc = Error_Resource;
		}
	}
	audiocontext_add_driver(media->context, &media->bridge->base);
	
	return rc;
}

static int phonemedia_start(audio_session_t *session)
{
	LOG_FUNCTION();
	audio_session_t *media = session;
	audio_session_phonemedia_conf_t *conf = &media->phonemedia_conf;
	apr_status_t status;
	int rc;
	int opt_micspk;
	const char *in_dev;
	const char *out_dev;
	int in_agc;
	int out_agc;
	int in_ns;
	int out_ns;
	int aec;
	const char *codec;

	in_dev = &session->conf.in_dev[conf->dev_type][0];
	out_dev = &session->conf.out_dev[conf->dev_type][0];
	Dbg("in_dev = %s,out_dev = %s, conf dir = %d.",in_dev,out_dev, conf->dir);

	opt_micspk = AMS_OPT_AS_STREAM;
	if (conf->dir & DIR_TX) {
		opt_micspk |= AMS_OPT_PLAY;
	}
	if (conf->dir &DIR_RX) {
		opt_micspk |= AMS_OPT_RECORD;
	}

	in_agc = media->conf.agc_in[conf->dev_type];
	out_agc = media->conf.agc_out[conf->dev_type];
	in_ns = media->conf.ns_in[conf->dev_type];
	out_ns = media->conf.ns_out[conf->dev_type];
	aec = media->conf.aec[conf->dev_type];

	switch (conf->local_pt) {
	case 0:
		codec = "PCMU";
		if (conf->local_ptime == 0)
			conf->local_ptime = 20;
		if (conf->remote_ptime == 0)
			conf->remote_ptime = 20;
		break;
	case 8: 
		codec = "PCMA";
		if (conf->local_ptime == 0)
			conf->local_ptime = 20;
		if (conf->remote_ptime == 0)
			conf->remote_ptime = 20;
		break;
	case 18:
		codec = "G729";
		if (conf->local_ptime == 0)
			conf->local_ptime = 20;
		if (conf->remote_ptime == 0)
			conf->remote_ptime = 20;
		break;
	default:
		codec = NULL;
		break;
	}
	if (codec == NULL)
		goto on_error;
	//assert(conf->local_ptime == conf->remote_ptime);
	if (conf->local_ptime != conf->remote_ptime) {
		conf->local_ptime = conf->remote_ptime;
	}

	status = apr_pool_create(&media->pool, NULL);
	if (status != APR_SUCCESS) {
		LogWarn(Severity_Middle, Error_Debug, ERROR_MOD_SIP_APR_POOL_CREATE_FAILED, "create media pool failed!");
		return Error_Resource;
	}

	status = audioengine_create(media->pool, &media->engine);
	if (status != APR_SUCCESS) {
		LogWarn(Severity_Middle, Error_Debug, ERROR_MOD_SIP_AUDIO_ENGINE_CREATE_FAILED, "create audio engine failed!");
		goto on_error;
	}

	status = audioengine_start(media->engine);
	if (status != APR_SUCCESS) {
		LogWarn(Severity_Middle, Error_Debug, ERROR_MOD_SIP_AUDIO_ENGINE_START_FAILED, "audio engine  start failed!");
		goto on_error;
	}
	Dbg("audioengine_start success!");

	rc = rtp_session_create2(conf->local_rtp_ip, conf->local_rtp_port, 2, &media->rtpsess);
	if (rc != 0) {
		char strmsg[MAX_PATH] = { 0 };
		_snprintf(strmsg, MAX_PATH, "audio rtp session create2 failed and local port is %d, rc=%d", conf->local_rtp_port, rc);
		LogWarn(Severity_Middle, Error_Debug, ERROR_MOD_SIP_AUDIO_RTP_SESSION_CREATE_FAILED, strmsg);
		LogWarn(Severity_Middle, Error_InvalidState, EVENT_MOD_SIP_AUDIO_RTP_CREATE, strmsg);
		goto on_error;
	}
	else{
		char strMessage[MAX_PATH] = { 0 };
		snprintf(strMessage, MAX_PATH, "audio rtp create2 success and local port is %d.", conf->local_rtp_port);
		LogWarn(Severity_Middle, Error_Debug, EVENT_MOD_SIP_AUDIO_RTP_CREATE, strMessage);
	}

	rc = rtp_session_reset2(media->rtpsess, conf->dir, conf->remote_rtp_ip, conf->remote_rtp_port, conf->remote_rtp_port + 1);
	if (rc != 0) {
		LogWarn(Severity_Middle, Error_Debug, ERROR_MOD_SIP_AUDIO_RTP_SESSION_RESET_FAILED, "audio rtp session reset failed!");
		goto on_error;
	}
	Dbg("rtp_session_reset2 success!");
	
	status = audiobridge_create(media->pool, media->engine, &media->bridge);
	if (status != APR_SUCCESS){
		LogWarn(Severity_Middle, Error_Debug, ERROR_MOD_SIP_AUDIO_BRIDGE_CREATE_FAILED, "audio bridge create failed!");
		goto on_error;
	}
	Dbg("audiobridge_create success!");

	status = apr_pool_create(&media->micspk_pool, media->pool);
	if (status != APR_SUCCESS){
		LogWarn(Severity_Middle, Error_Debug, ERROR_MOD_SIP_APR_POOL_CREATE_FAILED, "create media micspk_pool failed!");
		goto on_error;
	}
	Dbg("apr_pool_create success!");

	if (in_agc)
		opt_micspk |= AMS2_OPT_AGC;
	if (in_ns)
		opt_micspk |= AMS2_OPT_NS;
	if (aec)
		opt_micspk |= AMS2_OPT_AEC;
#ifdef RVC_OS_WIN
	//create Spk 
	status = audiomicspk2_create(media->micspk_pool, media->engine, opt_micspk, AUDIO_CLOCK, in_dev, out_dev, &media->micspkstream);
#else
	status = audiomicspkpulse_create(media->micspk_pool, media->engine, opt_micspk, AUDIO_CLOCK, in_dev, out_dev, conf->dev_type, &audio_device_event, &media->micspkstream);
#endif
	if (status != APR_SUCCESS)
		goto on_error;

	//音频回调
	Dbg("audiomicspk_create success!");
	media->micspkstream->user_data = media;
	media->micspkstream->on_rx_audio = &rx_audio_callback;
	media->micspkstream->on_tx_audio = &tx_audio_callback;
	media->micspkstream->on_audio_ns = &rvc_audio_ns;
	media->micspkstream->on_audio_play_ns = &rvc_audio_play_ns;
	//media->micspkstream->on_audio_device_event = &audio_device_event;
	Dbg("init on_rx_audio success!");

	if (out_agc || out_ns) {
		int read_opt = AUDIO_DSP_NONE;
		int write_opt = AUDIO_DSP_NONE;
		if (out_agc)
			write_opt |= AUDIO_DSP_AGC;
		if (out_ns)
			write_opt |= AUDIO_DSP_DENOISE;
		
		status = audiodsp_create(media->micspk_pool, media->engine, read_opt, write_opt, AUDIO_CLOCK, &media->dspstream);
		if (status != APR_SUCCESS){
			LogWarn(Severity_Middle, Error_Debug, ERROR_MOD_SIP_AUDIO_DSP_CREATE_FAILED, "create audio dsp failed!");
			goto on_error;
		}
		Dbg("audiodsp_create success!");
	}

#if 0
	status = audioaec_create(media->pool, media->engine, AUDIO_CLOCK, FRAME_TIME, 
		AUDIO_AEC_OPT_READ_AS_CAPTURE, &media->aecstream);
	if (status != APR_SUCCESS)
		goto on_error;
#endif

	status = audioresize_create(media->pool, media->engine, FRAME_TIME*2*AUDIO_CLOCK/1000, 
		conf->remote_ptime*2*AUDIO_CLOCK/1000, FRAME_TIME*2*AUDIO_CLOCK/1000, 
		conf->local_ptime*2*AUDIO_CLOCK/1000, &media->resizestream);
	if (status != APR_SUCCESS){
		LogWarn(Severity_Middle, Error_Debug, ERROR_MOD_SIP_AUDIO_RESIZE_CREATE_FAILED, "create audio resize failed!");
		goto on_error;
	}

	status = audiocodec_create(media->pool, media->engine, codec, AUDIO_CLOCK, FRAME_TIME, 
		AUDIO_CODEC_OPT_ENCODE_WRITE, &media->codecstream);
	if (status != APR_SUCCESS){
		LogWarn(Severity_Middle, Error_Debug, ERROR_MOD_SIP_AUDIO_CODEC_CREATE_FAILED, "create audio codec failed!");
		goto on_error;
	}

	status = audiortp_create(media->pool, media->engine, media->rtpsess, &media->rtpstream);
	if (status != APR_SUCCESS){
		LogWarn(Severity_Middle, Error_Debug, ERROR_MOD_SIP_AUDIO_RTP_CREATE_FAILED, "create audio rtp failed!");
		goto on_error;
	}
	Dbg("audio_session_t session addr is 0x%08x, audiortp_create success,and media->rtpstream addrs is 0x%08x!",session, media->rtpstream);

	g_nAudioRecvNum = 0;
	g_nAudioSendNum = 0;
	{
		int param;
		param = AUDIO_CLOCK;
		audiortp_set_param(media->rtpstream, AUDIO_RTP_FLAG_SEND_CLOCK, &param);
		audiortp_set_param(media->rtpstream, AUDIO_RTP_FLAG_RECV_CLOCK, &param);
		param = conf->local_pt;
		audiortp_set_param(media->rtpstream, AUDIO_RTP_FLAG_SEND_PT, &param);
		param = conf->remote_pt;
		audiortp_set_param(media->rtpstream, AUDIO_RTP_FLAG_RECV_PT, &param);
		param = conf->local_dtmf_pt;
		audiortp_set_param(media->rtpstream, AUDIO_RTP_FLAG_SEND_DTMF, &param);
		param = conf->remote_dtmf_pt;
		audiortp_set_param(media->rtpstream, AUDIO_RTP_FLAG_RECV_DTMF, &param);
		param = conf->local_ptime;
		audiortp_set_param(media->rtpstream, AUDIO_RTP_FLAG_SEND_PTIME, &param);
		param = conf->remote_ptime;
		audiortp_set_param(media->rtpstream, AUDIO_RTP_FLAG_RECV_PTIME, &param);
		//media->rtpstream->m_on_send_hook = &m_on_recv_hook;
		//Dbg("recv hook addr %d,send hook addr %d", &m_on_recv_hook,&m_on_send_hook);
		audiortp_set_param(media->rtpstream, AUDIO_RTP_FLAG_HOOK_ARG, media);
		Dbg("AUDIO_RTP_FLAG_HOOK_ARG addr is 0x%08x.", media);
		audiortp_set_param(media->rtpstream, AUDIO_RTP_FLAG_RECV_HOOK, (const void*)&recv_hook_callback);
		audiortp_set_param(media->rtpstream, AUDIO_RTP_FLAG_SEND_HOOK, (const void*)&send_hook_callback);
		//audiortp_set_param(media->rtpstream, AUDIO_RTP_FLAG_HOOK_ARG, media->rtpstream);
		audiortp_init(media->rtpstream);
	}

	if (conf->dir == DIR_TX) 
	{
		if (media->dspstream) 
		{
			audiostream_connect_pipeline(STREAM_DIR_READ, &media->micspkstream->base, &media->dspstream->base, NULL);
		} 
		else 
		{
			audiostream_connect_pipeline(STREAM_DIR_READ, &media->micspkstream->base, NULL);
		}
		audiobridge_set_leg(media->bridge, AUDIO_BRIDGE_LEG_LEFT, &media->micspkstream->base);
		audiostream_connect_pipeline(STREAM_DIR_WRITE, &media->resizestream->base, &media->codecstream->base, &media->rtpstream->base, NULL);
		audiobridge_set_leg(media->bridge, AUDIO_BRIDGE_LEG_RIGHT, &media->resizestream->base);
	} 
	else if (conf->dir == DIR_RX) 
	{
		if (media->dspstream) 
		{
			audiostream_connect_pipeline(STREAM_DIR_WRITE, &media->micspkstream->base, &media->dspstream->base, NULL);
		} 
		else 
		{
			audiostream_connect_pipeline(STREAM_DIR_WRITE, &media->micspkstream->base, NULL);
		}
		audiostream_connect_pipeline(STREAM_DIR_READ, &media->resizestream->base, &media->codecstream->base, &media->rtpstream->base, NULL);
		audiobridge_set_leg(media->bridge, AUDIO_BRIDGE_LEG_LEFT, &media->micspkstream->base);
		audiobridge_set_leg(media->bridge, AUDIO_BRIDGE_LEG_RIGHT, &media->resizestream->base);
	} 
	else 
	{
		if (media->dspstream) 
		{
			audiostream_connect_pipeline(STREAM_DIR_BOTH, &media->micspkstream->base, &media->dspstream->base, NULL);
		} 
		else 
		{
			audiostream_connect_pipeline(STREAM_DIR_BOTH, &media->micspkstream->base, NULL);
		}
		audiobridge_set_leg(media->bridge, AUDIO_BRIDGE_LEG_LEFT, &media->micspkstream->base);
		audiostream_connect_pipeline(STREAM_DIR_BOTH, &media->resizestream->base, &media->codecstream->base, &media->rtpstream->base, NULL);
		audiobridge_set_leg(media->bridge, AUDIO_BRIDGE_LEG_RIGHT, &media->resizestream->base);
	}

	status = audiocontext_create(media->pool, media->engine, &media->context);
	if (status != APR_SUCCESS){
		LogWarn(Severity_Middle, Error_Debug, ERROR_MOD_SIP_AUDIO_CONTEXT_CREATE_FAILED, "create audio context failed!");
		goto on_error;
	}
	Dbg("audiocontext_create success!");

	audiocontext_add_driver(media->context, &media->bridge->base);
	audioengine_start_context(media->engine, media->context);

	return 0;

on_error:
	phonemedia_stop(media, TRUE);
	Dbg("phonemedia_stop error!!!");
	return Error_Resource;
}

static int phonemedia_stop(audio_session_t *session, int b_record_turn_off)
{
	audio_session_t *media = session;
	if (media->context) {
		audioengine_stop_context(media->engine, media->context);
		audiocontext_remove_driver(media->context, &media->bridge->base);
		audiocontext_destroy(media->context);
		media->context = NULL;
	}
	if (media->engine) {
		audioengine_stop(media->engine);
		audioengine_destroy(media->engine);
		media->engine = NULL;
	}
	if (media->bridge) {
		audiobridge_destroy(media->bridge);
		media->bridge = NULL;
	}
	if (media->resizestream) {
		audioresize_destroy(media->resizestream);
		media->resizestream = NULL;
	}
	if (media->codecstream) {
		audiocodec_destroy(media->codecstream);
		media->codecstream = NULL;
	}
	if (media->rtpstream) {
		audiortp_destroy(media->rtpstream);
		media->rtpstream = NULL;
	}
	if (media->rtpsess) {
		unsigned short ilocal_port = 0;
		rtp_session_get_local_rtp_port(media->rtpsess, &ilocal_port);
		rtp_session_destroy(media->rtpsess);
		{
			char strInfo[MAX_PATH] = { 0 };
			snprintf(strInfo, MAX_PATH, "audio rtp(media->rtpsess) destroy and local port is %u.", ilocal_port);
			LogWarn(Severity_Middle, Error_Debug, EVENT_MOD_SIP_AUDIO_RTP_DESTROY, strInfo);
		}
		media->rtpsess = NULL;
	}
	//if (media->aecstream) {
	//	audioaec_destroy(media->aecstream);
	//	media->aecstream = NULL;
	//}
	if (media->dspstream) {
		audiodsp_destroy(media->dspstream);
		media->dspstream = NULL;
	}
	if (media->micspkstream) {
#ifdef _WIN32
		audiomicspk2_destroy(media->micspkstream);
#else
		audiomicspkpulse_destroy(media->micspkstream);
#endif
		Dbg("audiomicspk_destroy success!");
		media->micspkstream = NULL;
		apr_pool_destroy(media->micspk_pool);
		media->micspk_pool = NULL;
	}
	if (media->record.state) {
		if (media->record.codecstream) {
			audiocodec_destroy(media->record.codecstream);
			media->record.codecstream = NULL;
		}
		if (media->record.resizestream) {
			audioresize_destroy(media->record.resizestream);
			media->record.resizestream = NULL;
		}
		if (media->record.rtpstream) {
			audiortp_destroy(media->record.rtpstream);
			media->record.rtpstream = NULL;
			unsigned short ilocal_port = 0;
			rtp_session_get_local_rtp_port(media->record.rtpsess, &ilocal_port);
			rtp_session_destroy(media->record.rtpsess);
			{
				char strInfo[MAX_PATH] = { 0 };
				snprintf(strInfo, MAX_PATH, "audio rtp(record.rtpsess) destroy and local port is %u.", ilocal_port);
				LogWarn(Severity_Middle, Error_Debug, EVENT_MOD_SIP_AUDIO_RTP_DESTROY, strInfo);
			}
			media->record.rtpsess = NULL;
		}
		if (b_record_turn_off) {
			media->record.state = 0;
		}
	}
	if (media->pool) {
		apr_pool_destroy(media->pool);
		media->pool = NULL;
	}
	return 0;
}

static int phonemedia_chang_dev(audio_session_t *session, e_dev_type t)
{
	audio_session_t *media = session;
	audio_session_phonemedia_conf_t *conf = &media->phonemedia_conf;
	int opt_micspk;
	const char *in_dev;
	const char *out_dev;
	int in_agc;
	int out_agc;
	int in_ns;
	int out_ns;
	int aec;

	in_dev = &session->conf.in_dev[t][0];
	out_dev = &session->conf.out_dev[t][0];

	in_agc = media->conf.agc_in[t];
	out_agc = media->conf.agc_out[t];
	in_ns = media->conf.ns_in[t];
	out_ns = media->conf.ns_out[t];
	aec = media->conf.aec[t];

	opt_micspk = AMS_OPT_AS_STREAM;
	if (conf->dir & DIR_TX) {
		opt_micspk |= AMS_OPT_PLAY;
	}
	if (conf->dir &DIR_RX) {
		opt_micspk |= AMS_OPT_RECORD;
	}
	
	if (media->pool) {
		//apr_status_t status;
		audiocontext_remove_driver(media->context, &media->bridge->base);
		if (media->micspkstream) {
			Dbg("start audiomicspk_destroy");
#ifdef _WIN32
			audiomicspk2_destroy(media->micspkstream);
#else
			audiomicspkpulse_destroy(media->micspkstream);
#endif
			Dbg("audiomicspk_destroy success!");
			media->micspkstream = NULL;
		}
		if (media->dspstream) {
			audiodsp_destroy(media->dspstream);
			media->dspstream = NULL;
		}
		apr_pool_destroy(media->micspk_pool);
		apr_pool_create(&media->micspk_pool, media->pool);
		if (in_agc)
			opt_micspk |= AMS2_OPT_AGC;
		if (in_ns)
			opt_micspk |= AMS2_OPT_NS;
		if (aec)
			opt_micspk |= AMS2_OPT_AEC;

		Dbg("start audiomicspk_create");
#ifdef _WIN32
		audiomicspk2_create(media->micspk_pool, media->engine, opt_micspk, AUDIO_CLOCK, in_dev, out_dev, &media->micspkstream);
#else
		audiomicspkpulse_create(media->micspk_pool, media->engine, opt_micspk, AUDIO_CLOCK, in_dev, out_dev, t, &audio_device_event, &media->micspkstream);
#endif
		Dbg("start audiomicspk_create success");
		media->micspkstream->user_data = media;
		media->micspkstream->on_rx_audio = &rx_audio_callback;
		media->micspkstream->on_tx_audio = &tx_audio_callback;
		media->micspkstream->on_audio_ns = &rvc_audio_ns;
		media->micspkstream->on_audio_play_ns = &rvc_audio_play_ns;
		//media->micspkstream->on_audio_device_event = &audio_device_event;
		media->phonemedia_conf.dev_type = t;
		Dbg("init change dev on_rx_audio success!");
		
		if (out_agc || out_ns) {
			int read_opt = AUDIO_DSP_NONE;
			int write_opt = AUDIO_DSP_NONE;
			if (out_agc)
				write_opt |= AUDIO_DSP_AGC;
			if (out_ns)
				write_opt |= AUDIO_DSP_DENOISE;
			audiodsp_create(media->micspk_pool, media->engine, read_opt, write_opt, AUDIO_CLOCK, &media->dspstream);		
		}
		if (conf->dir == DIR_TX) {
			if (media->dspstream) {
				audiostream_connect_pipeline(STREAM_DIR_READ, &media->micspkstream->base, &media->dspstream->base, NULL);
			} else {
				audiostream_connect_pipeline(STREAM_DIR_READ, &media->micspkstream->base, NULL);
			}
			audiobridge_set_leg(media->bridge, AUDIO_BRIDGE_LEG_LEFT, &media->micspkstream->base);
			audiostream_connect_pipeline(STREAM_DIR_WRITE, &media->resizestream->base, &media->codecstream->base, &media->rtpstream->base, NULL);
			audiobridge_set_leg(media->bridge, AUDIO_BRIDGE_LEG_RIGHT, &media->resizestream->base);
		} 
		else if (conf->dir == DIR_RX) {
			if (media->dspstream) {
				audiostream_connect_pipeline(STREAM_DIR_WRITE, &media->micspkstream->base, &media->dspstream->base, NULL);
			} else {
				audiostream_connect_pipeline(STREAM_DIR_WRITE, &media->micspkstream->base, NULL);
			}
			audiostream_connect_pipeline(STREAM_DIR_READ, &media->resizestream->base, &media->codecstream->base, &media->rtpstream->base, NULL);
			audiobridge_set_leg(media->bridge, AUDIO_BRIDGE_LEG_LEFT, &media->micspkstream->base);
			audiobridge_set_leg(media->bridge, AUDIO_BRIDGE_LEG_RIGHT, &media->resizestream->base);
		} 
		else {
			if (media->dspstream) {
				audiostream_connect_pipeline(STREAM_DIR_BOTH, &media->micspkstream->base, &media->dspstream->base, NULL);
			} else {
				audiostream_connect_pipeline(STREAM_DIR_BOTH, &media->micspkstream->base, NULL);
			}
			audiobridge_set_leg(media->bridge, AUDIO_BRIDGE_LEG_LEFT, &media->micspkstream->base);
			audiostream_connect_pipeline(STREAM_DIR_BOTH, &media->resizestream->base, &media->codecstream->base, &media->rtpstream->base, NULL);
			audiobridge_set_leg(media->bridge, AUDIO_BRIDGE_LEG_RIGHT, &media->resizestream->base);
		}

		audiortp_reset_jitter(media->rtpstream);
		Dbg("phonemedia_chang_dev start audiocontext_add_driver");
		audiocontext_add_driver(media->context, &media->bridge->base);
		Dbg("phonemedia_chang_dev start audiocontext_add_driver success");
		return 0;
	} else {
		return Error_NotInit;
	}
	
	return Error_Unexpect;
}

static int phonemedia_start_remote_recording(audio_session_t *session, const audio_session_remote_recording_conf_t *conf)
{
	audio_session_t *media = session;
	if (!media->record.state) {
		media->record.state = TRUE;
		memcpy(&media->record.conf, conf, sizeof(audio_session_remote_recording_conf_t));
		if (media->pool) {
			int rc = phonemedia_on_remote_recording(media);
			if (rc != 0) {
				Dbg("start remote recording failed!");
			}
			return rc;
		}
		return 0;
	} else {
		return Error_Duplication;
	}
}

///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
//// audio session

static void __audionslog(void* user_data, const char* fmt, va_list arg)
{
	vDbg(fmt, arg);
}


int audio_session_create(const audio_session_conf_t *conf, audio_session_t **p_session)
{
	audio_session_t *session = ZALLOC_T(audio_session_t);
	session->remoteaudioqueue = new Clibaudioqueue(REC_COMMON_REMOTEAUDIO_SHM_QUEUE);
	session->brtpinsertqueue = false;
	session->baudiorecved = false;
	audions_callback_t t_callback = { 0 };
	t_callback.debug = &__audionslog;
	session->audionsobj = CreateIAudioNsObj(&t_callback);
	if (NULL != session->audionsobj){
		session->audionsobj->SetNsParams(8000, 10, 2);
	}
	session->audioplaynsobj = CreateIAudioNsObj(&t_callback);
	if (NULL != session->audioplaynsobj) {
		session->audioplaynsobj->SetNsParams(8000, 10, 2);
	}

	if (session) {
		memcpy(&session->conf, conf, sizeof(audio_session_conf_t));
		*p_session = session;
		return 0;
	} else {
		return Error_Resource;
	}
}

int audio_session_start_phonemedia(audio_session_t *session, const audio_session_phonemedia_conf_t *conf)
{
	int rc;
	LOG_FUNCTION();
	if (!session)
		return Error_NotInit;
	if (session->pool) { // already started
		phonemedia_stop(session, FALSE);
	}
	phonemedia_reconfig(session, conf);
	rc = phonemedia_start(session);
	return rc;
}

int audio_session_start_remote_recording(audio_session_t *session, const audio_session_remote_recording_conf_t *conf)
{
	return phonemedia_start_remote_recording(session, conf);
}
int audio_session_change_dev(audio_session_t *session, e_dev_type t)
{
	return phonemedia_chang_dev(session, t);
}
int audio_session_stop(audio_session_t *session)
{
	LOG_FUNCTION();
	return phonemedia_stop(session, TRUE);
}
void audio_session_destroy(audio_session_t *session)
{
	if (session->remoteaudioqueue){
		delete session->remoteaudioqueue;
	}

	if (NULL != session->audionsobj) {
		DestroyIAudioNsObj(session->audionsobj);
		session->audionsobj = NULL;
	}

	if (NULL != session->audioplaynsobj) {
		DestroyIAudioNsObj(session->audioplaynsobj);
		session->audioplaynsobj = NULL;
	}

	assert(session->pool == NULL);
	free(session);
}


///////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

static void __stdcall __audio_log_func(int level, const char *s)
{
	Dbg(s);
}

int audio_lib_init()
{
	LOG_FUNCTION();
	audio_log_set_func(&__audio_log_func);
	int rc = audioframework_init();

	if (rc != 0) {
		return Error_Resource;
	}
	else {
		audio_log_set_func(NULL);
#ifdef RVC_OS_WIN
		int icnt, ocnt;
		rc = audio_get_dev_count(&icnt, &ocnt);
		if (rc == 0) {
			int i;
			Dbg("audio input devices(%d):", icnt);
			for (i = 0; i < icnt; ++i) {
				CSimpleStringA str = audio_get_dev_name(true, i);
				Dbg("%d = %s", i, (LPCSTR)str);
			}
			Dbg("audio output devices(%d):", ocnt);
			for (i = 0; i < ocnt; ++i) {
				CSimpleStringA str = audio_get_dev_name(false, i);
				Dbg("%d = %s", i, (LPCSTR)str);
			}
		}
#endif
		audio_log_set_func(&__audio_log_func);
	}

	return 0;
}

void audio_lib_deinit()
{
	LOG_FUNCTION();
	audioframework_term();
}


#ifdef RVC_OS_WIN

int audio_get_dev_count(int *in_cnt, int *out_cnt)
{
	int icnt = 0, ocnt = 0;
	int cnt = Pa_GetDeviceCount();
	for (int i = 0; i < cnt; ++i) {
		const PaDeviceInfo *info = Pa_GetDeviceInfo(i);
		if (info->maxInputChannels)
			icnt ++;
		if (info->maxOutputChannels)
			ocnt ++;
	}
	if (in_cnt)
		*in_cnt = icnt;
	if (out_cnt)
		*out_cnt = ocnt;
	return 0;
}

CSimpleStringA audio_get_dev_name(bool in_direction, int idx)
{
	//audio_log_set_func(NULL);
	int cnt = Pa_GetDeviceCount();
	int ii, i;
	for (i = 0, ii = 0; i < cnt; ++i) {
		const PaDeviceInfo *info = Pa_GetDeviceInfo(i);
		if (in_direction) {
			if (info->maxInputChannels) {
				if (idx == ii) {
					//audio_log_set_func(__audio_log_func);
					return CSimpleStringA(info->name);
				}
				ii++;
			}
		} else {
			if (info->maxOutputChannels) {
				if (idx == ii) {
					//audio_log_set_func(__audio_log_func);
					return CSimpleStringA(info->name);
				}
				ii++;
			}
		}
	}
	//audio_log_set_func(__audio_log_func);
	return CSimpleStringA();
}

int capture_get_audio_device_id(bool in_direction, const char *dev_name)
{
	int cnt = Pa_GetDeviceCount();
	int ii, i;
	for (i = 0, ii = 0; i < cnt; ++i) {
		const PaDeviceInfo *info = Pa_GetDeviceInfo(i);
		if (in_direction) {
			if (info->maxInputChannels) {
				if (strstr(info->name, dev_name) != NULL) {
					return ii;
				}
				ii++;
			}
		} else {
			if (info->maxOutputChannels) {
				if (strstr(info->name, dev_name) != NULL) {
					return ii;
				}
				ii++;
			}
		}
	}
	return -1;
}


int get_audio_dev_name(char*devname,char*in_dev,char*out_dev)
{
	int icnt, ocnt;
	int rc = audio_get_dev_count(&icnt, &ocnt);
	//Dbg("get audio input num=%d,output num=%d",icnt,ocnt);
	if (rc == 0) 
	{
		int i;
		CSimpleStringA tmp;
		for (i = 0; i < icnt; ++i) 
		{		
			tmp = audio_get_dev_name(TRUE, i);
			//Dbg("input dev = %s!", tmp);
			if (strstr(tmp,devname)!=NULL)
			{
				//Dbg("get input dev = %s!", tmp);
				memcpy(in_dev,tmp.GetData(),tmp.GetLength());
				rc++;
			}
		}
		for (i = 0; i < ocnt; ++i) 
		{
			tmp = audio_get_dev_name(FALSE, i);
			//Dbg("output dev = %s!", tmp);
			if (strstr(tmp,devname)!=NULL)
			{
				//Dbg("get output dev = %s!", tmp);
				memcpy(out_dev,tmp.GetData(),tmp.GetLength());
				rc++;
			}
		}
		return rc;
	}
	else
	{
		return rc;
	}
}
#endif