myvtm/Module/mod_sipphone/video_render.cpp

#include "video_render.h"
#include "SpBase.h"
#include "Event.h"
#include "videoutil.h"
#include "libvideoqueue.h"
#include "rvc_media_common.h"
#include "fileutil.h"

extern "C"{
	#include <libavutil/avutil.h>
	#include <libavcodec/avcodec.h>
	#include <libswscale/swscale.h>
}
#include "video_common/ffmpeg_api_cpp_adapter.h"

#include "cv.h"
#include "highgui.h"
#include "y2k_time.h"


static int translate_image_resolution(video_frame** dstframe, const int iwidth, const int iheight, const video_frame* psrcframe)
{
	int i = -1;
	if (iwidth == psrcframe->width && iheight == psrcframe->height) {
		return i;
	}

	video_frame* pframe = video_frame_new(iwidth, iheight, VIDEO_FORMAT_RGB24);
	video_frame_fill_black(pframe);

	SwsContext* sws = sws_getCachedContext(NULL, psrcframe->width, psrcframe->height,
		PIX_FMT_BGR24,
		iwidth,
		iheight,
		PIX_FMT_BGR24,
		SWS_LANCZOS,
		NULL,
		NULL,
		NULL);

	sws_scale(sws, psrcframe->data, psrcframe->linesize, 0, psrcframe->height, pframe->data, pframe->linesize);
	sws_freeContext(sws);

	*dstframe = pframe;

	i = 0;

	return i;
}


static int rvc_video_shm_enqueue(Clibvideoqueue* shm_queue, video_frame* frame, int flags, int iframeid)
{
	videoq_frame tmp_frm;
	tmp_frm.data = frame->data[0];
	tmp_frm.framesize = frame->width * frame->height * 3;
	tmp_frm.format = VIDEOQ_FORMAT_RGB24;
	tmp_frm.width = frame->width;
	tmp_frm.height = frame->height;
	tmp_frm.iframeid = iframeid;
	unsigned int nowtime = y2k_time_now();
	if (!shm_queue->InsertVideo(&tmp_frm, flags, nowtime)){
		DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("caution: rvc render shm_queue video insert shm video failed!");
		return Error_Unexpect;
	}
	else
	{
		//DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("video render shm_queue video insert shm video ok, and nowtime is %d, and frame id is %d.", nowtime, iframeid);
		return Error_Succeed;
	}
}

static int get_local_video_frame(video_frame** frame, int itype, Clibvideoqueue* local_video_queue, int iwidth, int iheight, IplImage* personimage, IplImage* personmask)
{
	video_frame* tmp_frame_preview = NULL;
	tmp_frame_preview = video_frame_new(iwidth, iheight, VIDEO_FORMAT_RGB24);
	videoq_frame frm;
	frm.data = tmp_frame_preview->data[0];

	static bool blog = true;
	if (blog) {
		int ivideo_width = 0;
		int ivideo_height = 0;
		int isize = local_video_queue->GetFrameSize(ivideo_width, ivideo_height);
		DbgWithLink(LOG_LEVEL_DEBUG, LOG_TYPE_SYSTEM)("isize = %d, ivideo_width = %d, ivideo_height = %d.", isize, ivideo_width, ivideo_height);
		blog = false;
	}

	//DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("%s:%d, session->video_shm_q_preview = 0x%08x.", __FUNCTION__, __LINE__, local_video_queue);
	bool result = local_video_queue->GetVideo(&frm, VIDEOQUEUE_FLAG_HORIZONTAL_FLIP);
	if (result){
		if (1 == itype) {
			if (NULL != personimage && NULL != personmask)
			{
				IplImage* img = cvCreateImageHeader(cvSize(frm.width, frm.height), IPL_DEPTH_8U, 3);
				img->imageData = (char*)frm.data;
				if (frm.width != personimage->width) {
					IplImage* tmp = cvCreateImage(cvSize(frm.width, frm.height), IPL_DEPTH_8U, 3);
					IplImage* tmpmask = cvCreateImage(cvSize(frm.width, frm.height), IPL_DEPTH_8U, 1);
					cvResize(personimage, tmp);
					cvResize(personmask, tmpmask);
					cvAdd(img, tmp, img, tmpmask);
					cvReleaseImage(&tmp);
					cvReleaseImage(&tmpmask);
				}
				else {
					cvAdd(img, personimage, img, personmask);
				}
				cvReleaseImageHeader(&img);
			}
		}
		*frame = tmp_frame_preview;
	}

	return 0;
}


int rvc_remote_video_render(rvc_video_render_t* prender, void* videoframe)
{
	if (NULL != prender && NULL != videoframe) {
		if (NULL != prender->premote_render) {
			video_frame* echoframe = NULL;
			if (0 == translate_image_resolution(&echoframe, prender->render_param.iremote_view_cx, prender->render_param.iremote_view_cy, (const video_frame*)videoframe)) {
				prender->premote_render->RenderVideoFrame(echoframe, RVC_FLIP_VERTICAL);
				video_frame_delete(echoframe);
				echoframe = NULL;
			}
			else {
				prender->premote_render->RenderVideoFrame((video_frame*)videoframe, RVC_FLIP_VERTICAL);
			}
		}
	}

	return 0;
}

void* rvc_videorender_func(void* arg)
{
	rvc_video_render_t* param = (rvc_video_render_t*)arg;
	int ilocal_video_fresh_time = param->render_param.ilocal_fresh_time;

	int irender_camera = CAMERA_TYPE_ENV;
	if (param->render_param.ilocal_view_cx < param->render_param.ilocal_view_cy) {
		irender_camera = CAMERA_TYPE_OPT;
	}

	Clibvideoqueue* local_video_queue = NULL;
	RVC_RendererFlip renderflip = RVC_FLIP_VERTICAL;
	int iwidth = 0;
	int iheight = 0;
	if (CAMERA_TYPE_ENV == irender_camera) {
		local_video_queue = new Clibvideoqueue(REC_COMMON_VIDEO_ENV_SHM_PREVIEW_QUEUE);
		iwidth = REC_COMMON_VIDEO_PREVIEW_WIDTH;
		iheight = REC_COMMON_VIDEO_PREVIEW_HEIGHT;
	}
	else {
		local_video_queue = new Clibvideoqueue(REC_COMMON_VIDEO_OPT_SHM_PREVIEW_QUEUE);
		iwidth = REC_COMMON_VIDEO_PREVIEW_HEIGHT;
		iheight = REC_COMMON_VIDEO_PREVIEW_WIDTH;
		renderflip = RVC_FLIP_NONE;
	}

	int ivideo_width = 0;
	int ivideo_height = 0;
	int isize = local_video_queue->GetFrameSize(ivideo_width, ivideo_height);
	DbgWithLink(LOG_LEVEL_DEBUG, LOG_TYPE_SYSTEM)("%s:%d isize = %d, ivideo_width = %d, ivideo_height = %d.",__FUNCTION__, __LINE__, isize, ivideo_width, ivideo_height);
	if (ivideo_width > 0 && ivideo_height > 0) {
		iwidth = ivideo_width;
		iheight = ivideo_height;
	}

	char strPersonPath[MAX_PATH] = { 0 };
	snprintf(strPersonPath, MAX_PATH, "./bin/rxk.jpg");
	IplImage* personimage = NULL;
	IplImage* personmask = NULL;

	if (ExistsFile(strPersonPath)){
		personimage = cvLoadImage(strPersonPath);
		personmask = cvLoadImage(strPersonPath, 0);
	}

	if (NULL != param->plocal_render) {
		videorender_param_t tparam = { 0 };
		tparam.icx = param->render_param.ilocal_view_x;
		tparam.icy = param->render_param.ilocal_view_y;
		tparam.uwidth = param->render_param.ilocal_view_cx;
		tparam.uheight = param->render_param.ilocal_view_cy;
		tparam.ivideoformat = VIDEO_FORMAT_RGB24;
		if (0 == param->plocal_render->VideoRenderSetParam(&tparam)) {
			param->plocal_render->HideVideoWindow();
		}
		else {
			return 0;
		}
	}

	if (0 != param->render_param.iremote_view_cx && 0 != param->render_param.iremote_view_cy) {
		if (NULL != param->premote_render) {
			videorender_param_t tparam_remote = { 0 };
			tparam_remote.icx = param->render_param.iremote_view_x;
			tparam_remote.icy = param->render_param.iremote_view_y;
			tparam_remote.uwidth = param->render_param.iremote_view_cx;
			tparam_remote.uheight = param->render_param.iremote_view_cy;
			tparam_remote.ivideoformat = VIDEO_FORMAT_RGB24;
			if (0 != param->premote_render->VideoRenderSetParam(&tparam_remote)) {
				DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("video render set param failed.");
			}
		}
		else {
			DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("remote video render is null.");
		}
	}

	if (NULL != param->plocal_render) {
		bool bshow_local = false;
		bool bset = true;

		param->plocal_render->StartVideoRender();
		ivideo_width = 0;
		ivideo_height = 0;
		isize = local_video_queue->GetFrameSize(ivideo_width, ivideo_height);
		DbgWithLink(LOG_LEVEL_DEBUG, LOG_TYPE_SYSTEM)("%s:%d isize = %d, ivideo_width = %d, ivideo_height = %d.", __FUNCTION__, __LINE__, isize, ivideo_width, ivideo_height);

		for (; ; ) {
			struct timespec ts;
			clock_gettime(CLOCK_REALTIME, &ts);
			long unsec = ts.tv_nsec + (1000 * 1000 * ilocal_video_fresh_time);
			ts.tv_sec += (unsec / 1000000000);
			ts.tv_nsec = (unsec % 1000000000);
			if (0 != sem_timedwait(&param->local_render_stop_sem, &ts) && (ETIMEDOUT == errno))
			{
				video_frame* local_video_frame = NULL;
				int iwindowstate = param->cb.on_window_type(param->cb.user_data);
				get_local_video_frame(&local_video_frame, iwindowstate, local_video_queue, iwidth, iheight, personimage, personmask);

				if (NULL != local_video_frame) {
					video_frame* localframe = NULL;
					if (0 == translate_image_resolution(&localframe, param->render_param.ilocal_view_cx, param->render_param.ilocal_view_cy, local_video_frame)) {					
						param->plocal_render->RenderVideoFrame(localframe, renderflip);
						video_frame_delete(localframe);
						localframe = NULL;
					}
					else {
						param->plocal_render->RenderVideoFrame(local_video_frame, renderflip);
					}

					bshow_local = true;
					video_frame_delete(local_video_frame);
					local_video_frame = NULL;
				}
			}
			else {
				param->plocal_render->HideVideoWindow();
				break;
			}

			if (bset) {
				if (bshow_local) {
					param->plocal_render->ShowVideoWindow();
					param->premote_render->ShowVideoWindow();
					bset = false;
				}
			}
		}
		param->plocal_render->StopVideoRender();
	}

	if (NULL != personimage) {
		cvReleaseImage(&personimage);
	}

	if (NULL != personmask) {
		cvReleaseImage(&personmask);
	}

	return 0;
}


void* rvc_remote_videorender_func(void* arg)
{
	rvc_video_render_t* param = (rvc_video_render_t*)arg;
	int iremote_video_fresh_time = param->render_param.iremote_fresh_time;
	char strImgPath[MAX_PATH] = { 0 };
	int irecord_video_frame_width = REC_COMMON_VIDEO_SNAPSHOT_PREVIEW_WIDTH;
	int irecord_video_frame_heigt = REC_COMMON_VIDEO_SNAPSHOT_PREVIEW_HEIGHT;

	Clibvideoqueue* video_shm_q_remote = new Clibvideoqueue(REC_COMMON_VIDEO_REMOTE_SHM_RTP_QUEUE);
	
	snprintf(strImgPath, MAX_PATH, "%s", "./bin/agent.jpg");

	video_frame* remote_frame = video_frame_new(irecord_video_frame_width, irecord_video_frame_heigt, VIDEO_FORMAT_RGB24);
	video_frame_fill_black(remote_frame);

	if (ExistsFile(strImgPath)){
		IplImage* img = cvLoadImage(strImgPath, 1);
		videoq_frame* vframe = new videoq_frame;

		if (NULL != img) {
			vframe->format = VIDEOQ_FORMAT_RGB24;
			vframe->framesize = img->imageSize;
			vframe->width = img->width;
			vframe->height = img->height;
			vframe->data = new unsigned char[img->imageSize];
			memcpy(vframe->data, img->imageData, img->imageSize);
			cvReleaseImage(&img);
		}

		SwsContext* sws = sws_getContext(vframe->width, vframe->height, PIX_FMT_BGR24,
			irecord_video_frame_width,
			irecord_video_frame_heigt,
			PIX_FMT_BGR24,
			SWS_POINT, NULL, NULL, NULL);

		uint8_t* src_data[4] = { (unsigned char*)vframe->data + (vframe->height - 1) * vframe->width * 3, NULL, NULL, NULL };
		int src_linesize[4] = { -vframe->width * 3, 0, 0, 0};
		unsigned char* dst[4] = { remote_frame->data[0],NULL,NULL,NULL };
		int dst_linesize[4] = { remote_frame->linesize[0], 0, 0, 0};
		sws_scale(sws, src_data, src_linesize, 0, vframe->height, dst, dst_linesize);
		sws_freeContext(sws);

		if (vframe) {
			delete vframe->data;
			delete vframe;
			vframe = NULL;
		}
	}

	video_frame* showframe = NULL;
	if (-1 == translate_image_resolution(&showframe, param->render_param.iremote_view_cx, param->render_param.iremote_view_cy, remote_frame)) {
		showframe = video_frame_new(param->render_param.iremote_view_cx, param->render_param.iremote_view_cy, VIDEO_FORMAT_RGB24);
		video_frame_fill_black(showframe);
		video_frame_copy(showframe, remote_frame);
	}

	int iframeid = 0;
	int rc = rvc_video_shm_enqueue(video_shm_q_remote, remote_frame, VIDEOQUEUE_FLAG_VERTICAL_FLIP, iframeid++);
	if (rc != Error_Succeed) {
		DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("insert agent picture to remote video queue failed.");
	}

	for (; ; ) {
		struct timespec ts;
		clock_gettime(CLOCK_REALTIME, &ts);
		long unsec = ts.tv_nsec + (1000 * 1000 * iremote_video_fresh_time);
		ts.tv_sec += (unsec / 1000000000);
		ts.tv_nsec = (unsec % 1000000000);
		if (0 != sem_timedwait(&param->remote_render_stop_sem, &ts) && (ETIMEDOUT == errno)){
			rvc_remote_video_render(param, showframe);
			if (NULL != video_shm_q_remote){
				rc = rvc_video_shm_enqueue(video_shm_q_remote, remote_frame, VIDEOQUEUE_FLAG_VERTICAL_FLIP, iframeid++);
				if (rc != Error_Succeed){
					DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("insert agent picture to remote video queue failed.");
				}
			}
		}
		else {
			break;
		}
	}

	video_frame_delete(showframe);
	showframe = NULL;

	video_frame_delete(remote_frame);
	remote_frame = NULL;

	return 0;
}


int rvc_start_video_render(rvc_video_render_t* prender, bool bremote, rvc_video_box_move_callback_t* cb)
{
	if (0 != sem_init(&prender->local_render_stop_sem, 0, 0)) {
		DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("create local stop sem failed!");
		return Error_Resource;
	}

	int err = pthread_create(&prender->local_render_threadid, NULL, rvc_videorender_func, prender);
	if (Error_Succeed != err) {
		DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("create local video render thread failed.");
		return Error_Resource;
	}

	if (bremote) {
		if (0 != sem_init(&prender->remote_render_stop_sem, 0, 0)) {
			DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("create remote stop sem failed!");
			return Error_Resource;
		}

		err = pthread_create(&prender->remote_render_threadid, NULL, rvc_remote_videorender_func, prender);
		if (Error_Succeed != err) {
			DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("create remote video render thread failed.");
			return Error_Resource;
		}
	}

	return err;
}


int rvc_stop_video_render(rvc_video_render_t* prender)
{
	if (NULL == prender) {
		return Error_Param;
	}

	sem_post(&prender->local_render_stop_sem);
	rvc_stop_remote_video_render(prender);
	if (prender->local_render_threadid > 0) {
		if (0 == pthread_join(prender->local_render_threadid, NULL)) {
			prender->local_render_threadid = 0;
		}
		else {
			DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("video render thread pthread join error for %s.", strerror(errno));
		}
	}
	return Error_Succeed;
}

int rvc_stop_remote_video_render(rvc_video_render_t* prender)
{
	if (prender->remote_render_threadid > 0) {
		sem_post(&prender->remote_render_stop_sem);
		if (0 == pthread_join(prender->remote_render_threadid, NULL)) {
			prender->remote_render_threadid = 0;
		}
		else {
			DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("remote video render thread pthread join error for %s.", strerror(errno));
		}
	}

	return Error_Succeed;
}