myvtm/Module/mod_sipphone/video_render.cpp

#include "video_render.h"
#include "SpBase.h"
#include "Event.h"
#include "videoutil.h"
#include "libvideoqueue.h"
#include "rvc_media_common.h"
#include "fileutil.h"

#include "video_common/ffmpeg_api_cpp_adapter.h"

#include "cv.h"
#include "highgui.h"
#include "y2k_time.h"


static int translate_image_resolution(video_frame** dstframe, const int iwidth, const int iheight, const video_frame* psrcframe)
{
	int i = -1;
	if (iwidth == psrcframe->width && iheight == psrcframe->height) {
		return i;
	}

	video_frame* pframe = video_frame_new(iwidth, iheight, VIDEO_FORMAT_RGB24);
	video_frame_fill_black(pframe);

	SwsContext* sws = sws_getCachedContext(NULL, psrcframe->width, psrcframe->height,
		PIX_FMT_BGR24,
		iwidth,
		iheight,
		PIX_FMT_BGR24,
		SWS_LANCZOS,
		NULL,
		NULL,
		NULL);

	sws_scale(sws, psrcframe->data, psrcframe->linesize, 0, psrcframe->height, pframe->data, pframe->linesize);
	sws_freeContext(sws);

	*dstframe = pframe;

	i = 0;

	return i;
}


static int rvc_video_shm_enqueue(Clibvideoqueue* shm_queue, video_frame* frame, int flags, int iframeid)
{
	videoq_frame tmp_frm;
	tmp_frm.data = frame->data[0];
	tmp_frm.framesize = frame->width * frame->height * 3;
	tmp_frm.format = VIDEOQ_FORMAT_RGB24;
	tmp_frm.width = frame->width;
	tmp_frm.height = frame->height;
	tmp_frm.iframeid = iframeid;
	unsigned int nowtime = y2k_time_now();
	if (!shm_queue->InsertVideo(&tmp_frm, flags, nowtime)){
		DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("caution: rvc render shm_queue video insert shm video failed!");
		return Error_Unexpect;
	}
	else{
		//DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("video render shm_queue video insert shm video ok, and nowtime is %d, and frame id is %d.", nowtime, iframeid);
		return Error_Succeed;
	}
}

static bool get_local_video_frame(video_frame** frame, int itype, Clibvideoqueue* local_video_queue, int iwidth, int iheight, IplImage* personimage, IplImage* personmask)
{
	bool result = false;
	video_frame* tmp_frame_preview = NULL;
	tmp_frame_preview = video_frame_new(iwidth, iheight, VIDEO_FORMAT_RGB24);
	if (NULL == tmp_frame_preview) {
		return result;
	}
	videoq_frame frm;
	frm.data = tmp_frame_preview->data[0];

	static bool blog = true;
	if (blog) {
		int ivideo_width = 0;
		int ivideo_height = 0;
		int isize = local_video_queue->GetFrameSize(ivideo_width, ivideo_height);
		blog = false;
	}

	result = local_video_queue->GetVideo(&frm, VIDEOQUEUE_FLAG_HORIZONTAL_FLIP);
	if (result){
		if (1 == itype) {
			if (NULL != personimage && NULL != personmask)
			{
				IplImage* img = cvCreateImageHeader(cvSize(frm.width, frm.height), IPL_DEPTH_8U, 3);
				img->imageData = (char*)frm.data;
				if (frm.width != personimage->width) {
					IplImage* tmp = cvCreateImage(cvSize(frm.width, frm.height), IPL_DEPTH_8U, 3);
					IplImage* tmpmask = cvCreateImage(cvSize(frm.width, frm.height), IPL_DEPTH_8U, 1);
					cvResize(personimage, tmp);
					cvResize(personmask, tmpmask);
					cvAdd(img, tmp, img, tmpmask);
					cvReleaseImage(&tmp);
					cvReleaseImage(&tmpmask);
				}
				else {
					cvAdd(img, personimage, img, personmask);
				}
				cvReleaseImageHeader(&img);
			}
		}
		*frame = tmp_frame_preview;
	}
	else {
		video_frame_delete(tmp_frame_preview);
		tmp_frame_preview = NULL;
	}
	
	return result;
}


int rvc_remote_video_render(rvc_video_render_t* prender, void* videoframe)
{
	if (NULL != prender && NULL != videoframe) {
		if (NULL != prender->premote_render) {
			video_frame* echoframe = NULL;
			if (0 == translate_image_resolution(&echoframe, prender->render_param.iremote_view_cx, prender->render_param.iremote_view_cy, (const video_frame*)videoframe)) {
				prender->premote_render->RenderVideoFrame(echoframe, RVC_FLIP_VERTICAL);
				video_frame_delete(echoframe);
				echoframe = NULL;
			}
			else {
				prender->premote_render->RenderVideoFrame((video_frame*)videoframe, RVC_FLIP_VERTICAL);
			}
		}
	}

	return 0;
}


static int rvc_getrender_videoqueue(Clibvideoqueue** render_video_queue, int* render_width, int* render_height, RVC_RendererFlip* renderflip, rvc_video_render_t* param)
{
	int iret = -1;

	int irender_camera = CAMERA_TYPE_ENV;
	if (param->render_param.ilocal_view_cx < param->render_param.ilocal_view_cy) {
		irender_camera = CAMERA_TYPE_OPT;
	}

	if (CAMERA_TYPE_ENV == irender_camera) {
		if (ePreview_Queue == param->render_param.erender_queue) {
			*render_video_queue = new Clibvideoqueue(REC_COMMON_VIDEO_ENV_SHM_PREVIEW_QUEUE);
			*render_width = REC_COMMON_VIDEO_PREVIEW_WIDTH;
			*render_height = REC_COMMON_VIDEO_PREVIEW_HEIGHT;
		}
		else {
			*render_video_queue = new Clibvideoqueue(REC_COMMON_VIDEO_ENV_SHM_SNAPSHOT_QUEUE);
			*render_width = REC_COMMON_VIDEO_SNAPSHOT_WIDTH;
			*render_height = REC_COMMON_VIDEO_SNAPSHOT_HEIGHT;
			DbgWithLink(LOG_LEVEL_DEBUG, LOG_TYPE_SYSTEM)("use sanpshot queue.");
		}
#ifdef RVC_OS_WIN
		*renderflip = RVC_FLIP_HORIZONTAL;
#endif // RVC_OS_WIN
	}
	else {
		if (ePreview_Queue == param->render_param.erender_queue) {
			*render_video_queue = new Clibvideoqueue(REC_COMMON_VIDEO_OPT_SHM_PREVIEW_QUEUE);
			*render_width = REC_COMMON_VIDEO_PREVIEW_HEIGHT;
			*render_height = REC_COMMON_VIDEO_PREVIEW_WIDTH;
		}
		else {
			*render_video_queue = new Clibvideoqueue(REC_COMMON_VIDEO_OPT_SHM_SNAPSHOT_QUEUE);
			*render_width = REC_COMMON_VIDEO_SNAPSHOT_HEIGHT;
			*render_height = REC_COMMON_VIDEO_SNAPSHOT_WIDTH;
			DbgWithLink(LOG_LEVEL_DEBUG, LOG_TYPE_SYSTEM)("use sanpshot queue.");
		}
		*renderflip = RVC_FLIP_NONE;
	}

	iret = 0;

	return iret;
}


static int rvc_getrxkimage(IplImage** personimage, IplImage** personmask)
{
	int iret = -1;

	char strPersonPath[MAX_PATH] = { 0 };
	rvc_snprintf(strPersonPath, MAX_PATH, "./bin/rxk.jpg");

	if (ExistsFile(strPersonPath)) {
		*personimage = cvLoadImage(strPersonPath);
		*personmask = cvLoadImage(strPersonPath, 0);
		iret = 0;
	}

	return iret;
}


#ifdef RVC_OS_WIN
static unsigned int __stdcall rvc_videorender_func(void* arg)
#else
static void* rvc_videorender_func(void* arg)
#endif
{
	rvc_video_render_t* param = (rvc_video_render_t*)arg;
	int ilocal_video_fresh_time = param->render_param.ilocal_fresh_time;
	bool bremote_video = false;

	Clibvideoqueue* local_video_queue = NULL;
	RVC_RendererFlip renderflip = RVC_FLIP_VERTICAL;
	int iwidth = 0;
	int iheight = 0;

	rvc_getrender_videoqueue(&local_video_queue, &iwidth, &iheight, &renderflip, param);

	int ivideo_width = 0;
	int ivideo_height = 0;
	int isize = local_video_queue->GetFrameSize(ivideo_width, ivideo_height);
	
	if (ivideo_width > 0 && ivideo_height > 0) {
		iwidth = ivideo_width;
		iheight = ivideo_height;
	}

	IplImage* personimage = NULL;
	IplImage* personmask = NULL;
	rvc_getrxkimage(&personimage, &personmask);

	if (NULL != param->plocal_render) {
		videorender_param_t tparam = { 0 };
		tparam.icx = param->render_param.ilocal_view_x;
		tparam.icy = param->render_param.ilocal_view_y;
		tparam.uwidth = param->render_param.ilocal_view_cx;
		tparam.uheight = param->render_param.ilocal_view_cy;
		tparam.ivideoformat = VIDEO_FORMAT_RGB24;
		if (0 == param->plocal_render->VideoRenderSetParam(&tparam)) {
			param->plocal_render->HideVideoWindow();
		}
		else {
			return 0;
		}
	}

	if (0 != param->render_param.iremote_view_cx && 0 != param->render_param.iremote_view_cy) {
		if (NULL != param->premote_render) {
			videorender_param_t tparam_remote = { 0 };
			tparam_remote.icx = param->render_param.iremote_view_x;
			tparam_remote.icy = param->render_param.iremote_view_y;
			tparam_remote.uwidth = param->render_param.iremote_view_cx;
			tparam_remote.uheight = param->render_param.iremote_view_cy;
			tparam_remote.ivideoformat = VIDEO_FORMAT_RGB24;
			if (0 != param->premote_render->VideoRenderSetParam(&tparam_remote)) {
				DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("video render set param failed.");
			}
			bremote_video = true;
		}
		else {
			DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("remote video render is null.");
		}
	}

	if (NULL != param->plocal_render) {
		bool bshow_local = false;
		bool bset = true;

		param->plocal_render->StartVideoRender();
		ivideo_width = 0;
		ivideo_height = 0;
		isize = local_video_queue->GetFrameSize(ivideo_width, ivideo_height);

		for (; ; ) {
#ifdef RVC_OS_WIN
			DWORD dwRet = WaitForSingleObject(param->local_render_stop_event, ilocal_video_fresh_time);
			if (WAIT_TIMEOUT == dwRet) {
#else
			struct timespec ts;
			clock_gettime(CLOCK_REALTIME, &ts);
			long unsec = ts.tv_nsec + (1000 * 1000 * ilocal_video_fresh_time);
			ts.tv_sec += (unsec / 1000000000);
			ts.tv_nsec = (unsec % 1000000000);
			if (0 != sem_timedwait(&param->local_render_stop_sem, &ts) && (ETIMEDOUT == errno)){
#endif
				video_frame* local_video_frame = NULL;
				int iwindowstate = param->cb.on_window_type(param->cb.user_data);
				get_local_video_frame(&local_video_frame, iwindowstate, local_video_queue, iwidth, iheight, personimage, personmask);

				if (NULL != local_video_frame) {
					video_frame* localframe = NULL;
					if (0 == translate_image_resolution(&localframe, param->render_param.ilocal_view_cx, param->render_param.ilocal_view_cy, local_video_frame)) {					
						param->plocal_render->RenderVideoFrame(localframe, renderflip);
						video_frame_delete(localframe);
						localframe = NULL;
					}
					else {
						param->plocal_render->RenderVideoFrame(local_video_frame, renderflip);
					}

					bshow_local = true;
					video_frame_delete(local_video_frame);
					local_video_frame = NULL;
				}
			}
			else {
				param->plocal_render->HideVideoWindow();
				break;
			}

			if (bset) {
				if (bshow_local) {
					param->plocal_render->ShowVideoWindow();
					if (bremote_video) {
						param->premote_render->ShowVideoWindow();
					}
					bset = false;
				}
			}
		}
		param->plocal_render->StopVideoRender();
	}

	if (NULL != personimage) {
		cvReleaseImage(&personimage);
	}

	if (NULL != personmask) {
		cvReleaseImage(&personmask);
	}

	return 0;
}


static int rvc_get_remote_videoframe(video_frame** remote_frame)
{
	int iret = -1;

	char strImgPath[MAX_PATH] = { 0 };
	rvc_snprintf(strImgPath, MAX_PATH, "%s", "./bin/agent.jpg");
	int irecord_video_frame_width = REC_COMMON_VIDEO_SNAPSHOT_PREVIEW_WIDTH;
	int irecord_video_frame_heigt = REC_COMMON_VIDEO_SNAPSHOT_PREVIEW_HEIGHT;

	*remote_frame = video_frame_new(irecord_video_frame_width, irecord_video_frame_heigt, VIDEO_FORMAT_RGB24);
	video_frame_fill_black(*remote_frame);

	if (ExistsFileA(strImgPath)) {
		IplImage* img = cvLoadImage(strImgPath, 1);
		videoq_frame* vframe = new videoq_frame;

		if (NULL != img) {
			vframe->format = VIDEOQ_FORMAT_RGB24;
			vframe->framesize = img->imageSize;
			vframe->width = img->width;
			vframe->height = img->height;
			vframe->data = new unsigned char[img->imageSize];
			memcpy(vframe->data, img->imageData, img->imageSize);
			cvReleaseImage(&img);
		}

		SwsContext* sws = sws_getContext(vframe->width, vframe->height, PIX_FMT_BGR24,
			irecord_video_frame_width,
			irecord_video_frame_heigt,
			PIX_FMT_BGR24,
			SWS_POINT, NULL, NULL, NULL);

		uint8_t* src_data[4] = { (unsigned char*)vframe->data + (vframe->height - 1) * vframe->width * 3, NULL, NULL, NULL };
		int src_linesize[4] = { -vframe->width * 3, 0, 0, 0 };
		unsigned char* dst[4] = { (*remote_frame)->data[0],NULL,NULL,NULL };
		int dst_linesize[4] = { (*remote_frame)->linesize[0], 0, 0, 0 };
		sws_scale(sws, src_data, src_linesize, 0, vframe->height, dst, dst_linesize);
		sws_freeContext(sws);

		if (vframe) {
			delete vframe->data;
			delete vframe;
			vframe = NULL;
		}

		iret = 0;
	}

	return iret;
}


#ifdef RVC_OS_WIN
static unsigned int __stdcall rvc_remote_videorender_func(void* arg)
#else
static void* rvc_remote_videorender_func(void* arg)
#endif
{
	rvc_video_render_t* param = (rvc_video_render_t*)arg;
	int iremote_video_fresh_time = param->render_param.iremote_fresh_time;

	Clibvideoqueue* video_shm_q_remote = new Clibvideoqueue(REC_COMMON_VIDEO_REMOTE_SHM_RTP_QUEUE);

	video_frame* remote_frame = NULL;
	rvc_get_remote_videoframe(&remote_frame);

	video_frame* showframe = NULL;
	if (-1 == translate_image_resolution(&showframe, param->render_param.iremote_view_cx, param->render_param.iremote_view_cy, remote_frame)) {
		showframe = video_frame_new(param->render_param.iremote_view_cx, param->render_param.iremote_view_cy, VIDEO_FORMAT_RGB24);
		video_frame_fill_black(showframe);
		video_frame_copy(showframe, remote_frame);
	}

	int iframeid = 0;
	int rc = rvc_video_shm_enqueue(video_shm_q_remote, remote_frame, VIDEOQUEUE_FLAG_VERTICAL_FLIP, iframeid++);
	if (rc != Error_Succeed) {
		DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("insert agent picture to remote video queue failed.");
	}

	for (; ; ) {
#ifdef RVC_OS_WIN
		DWORD dwRet = WaitForSingleObject(param->remote_render_stop_event, iremote_video_fresh_time);
		if (WAIT_TIMEOUT == dwRet){
#else
		struct timespec ts;
		clock_gettime(CLOCK_REALTIME, &ts);
		long unsec = ts.tv_nsec + (1000 * 1000 * iremote_video_fresh_time);
		ts.tv_sec += (unsec / 1000000000);
		ts.tv_nsec = (unsec % 1000000000);
		if (0 != sem_timedwait(&param->remote_render_stop_sem, &ts) && (ETIMEDOUT == errno)){
#endif // RVC_OS_WIN
			rvc_remote_video_render(param, showframe);
			if (NULL != video_shm_q_remote){
				rc = rvc_video_shm_enqueue(video_shm_q_remote, remote_frame, VIDEOQUEUE_FLAG_VERTICAL_FLIP, iframeid++);
				if (rc != Error_Succeed){
					DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("insert agent picture to remote video queue failed.");
				}
			}
		}
		else {
			break;
		}
	}

	video_frame_delete(showframe);
	showframe = NULL;

	video_frame_delete(remote_frame);
	remote_frame = NULL;

	return 0;
}


int rvc_start_video_render(rvc_video_render_t* prender, bool bremote, rvc_video_box_move_callback_t* cb)
{
	int errcode = -1;
	if (NULL == prender) {
		return errcode;
	}

#ifdef RVC_OS_WIN
	prender->local_render_stop_event = CreateEventA(NULL, false, false, NULL);
	if (NULL == prender->local_render_stop_event) {
		DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("create local stop event failed!");
		return Error_Resource;
	}

	prender->local_render_thread = (HANDLE)_beginthreadex(NULL, 0, rvc_videorender_func, prender, 0, NULL);
	if (NULL == prender->local_render_thread) {
		DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("create local video render thread failed.");
		return Error_Resource;
	}

	if (bremote) {
		prender->remote_render_stop_event = CreateEventA(NULL, false, false, NULL);
		if (NULL == prender->local_render_stop_event) {
			DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("create remote stop event failed!");
			return Error_Resource;
		}

		prender->remote_render_thread = (HANDLE)_beginthreadex(NULL, 0, rvc_remote_videorender_func, prender, 0, NULL);
		if (NULL == prender->remote_render_thread) {
			DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("create remote video render thread failed.");
			return Error_Resource;
		}
	}
	errcode = 0;
#else
	if (0 != sem_init(&prender->local_render_stop_sem, 0, 0)) {
		DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("create local stop sem failed!");
		return Error_Resource;
	}

	errcode = pthread_create(&prender->local_render_threadid, NULL, rvc_videorender_func, prender);
	if (Error_Succeed != errcode) {
		DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("create local video render thread failed.");
		return Error_Resource;
	}

	if (bremote) {
		if (0 != sem_init(&prender->remote_render_stop_sem, 0, 0)) {
			DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("create remote stop sem failed!");
			return Error_Resource;
		}

		errcode = pthread_create(&prender->remote_render_threadid, NULL, rvc_remote_videorender_func, prender);
		if (Error_Succeed != errcode) {
			DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("create remote video render thread failed.");
			return Error_Resource;
		}
	}
#endif // RVC_OS_WIN

	return errcode;
}


int rvc_stop_video_render(rvc_video_render_t* prender)
{
	if (NULL == prender) {
		return Error_Param;
	}

#ifdef RVC_OS_WIN
	if (prender->local_render_stop_event) {
		SetEvent(prender->local_render_stop_event);
	}
	rvc_stop_remote_video_render(prender);
	if (NULL != prender->local_render_thread) {
		WaitForSingleObject(prender->local_render_thread, INFINITE);
		CloseHandle(prender->local_render_thread);
		prender->local_render_thread = NULL;
	}
#else
	sem_post(&prender->local_render_stop_sem);
	rvc_stop_remote_video_render(prender);
	if (prender->local_render_threadid > 0) {
		if (0 == pthread_join(prender->local_render_threadid, NULL)) {
			prender->local_render_threadid = 0;
		}
		else {
			DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("video render thread pthread join error for %s.", strerror(errno));
		}
	}
#endif // RVC_OS_WIN

	return Error_Succeed;
}

int rvc_stop_remote_video_render(rvc_video_render_t* prender)
{
#ifdef RVC_OS_WIN
	if (NULL != prender->remote_render_thread) {
		SetEvent(prender->remote_render_stop_event);
		WaitForSingleObject(prender->remote_render_thread, INFINITE);
		CloseHandle(prender->remote_render_thread);
		prender->remote_render_thread = NULL;
	}
#else
	if (prender->remote_render_threadid > 0) {
		sem_post(&prender->remote_render_stop_sem);
		if (0 == pthread_join(prender->remote_render_threadid, NULL)) {
			prender->remote_render_threadid = 0;
		}
		else {
			DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("remote video render thread pthread join error for %s.", strerror(errno));
		}
	}
#endif

	return Error_Succeed;
}