myvtm/Module/mod_sipphone/video_render.cpp

#include "video_render.h"
#include "SpBase.h"
#include "Event.h"
#include "../../Other/libvideoframework/videoutil.h"
#include "../../Other/libvideoqueue/libvideoqueue.h"
#include "../../Other/rvcmediacommon/rvc_media_common.h"
#include "fileutil.h"

extern "C"{
	#include <libavutil/avutil.h>
	#include <libavcodec/avcodec.h>
	#include <libswscale/swscale.h>
}
#include "../../Other/libvideoframework/video_common/ffmpeg_api_cpp_adapter.h"

#include "cv.h"
#include "highgui.h"
#include "y2k_time.h"


static int translate_image_resolution(video_frame** dstframe, const int iwidth, const int iheight, const video_frame* psrcframe)
{
	int i = -1;
	if (iwidth == psrcframe->width && iheight == psrcframe->height) {
		return i;
	}

	video_frame* pframe = video_frame_new(iwidth, iheight, VIDEO_FORMAT_RGB24);
	video_frame_fill_black(pframe);

	SwsContext* sws = sws_getCachedContext(NULL, psrcframe->width, psrcframe->height,
		PIX_FMT_BGR24,
		iwidth,
		iheight,
		PIX_FMT_BGR24,
		SWS_LANCZOS,
		NULL,
		NULL,
		NULL);

	sws_scale(sws, psrcframe->data, psrcframe->linesize, 0, psrcframe->height, pframe->data, pframe->linesize);
	sws_freeContext(sws);

	*dstframe = pframe;

	i = 0;

	return i;
}


static int rvc_video_shm_enqueue(Clibvideoqueue* shm_queue, video_frame* frame, int flags, int iframeid)
{
	videoq_frame tmp_frm;
	tmp_frm.data = frame->data[0];
	tmp_frm.framesize = frame->width * frame->height * 3;
	tmp_frm.format = VIDEOQ_FORMAT_RGB24;
	tmp_frm.width = frame->width;
	tmp_frm.height = frame->height;
	tmp_frm.iframeid = iframeid;
	unsigned int nowtime = y2k_time_now();
	if (!shm_queue->InsertVideo(&tmp_frm, flags, nowtime)){
		DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("caution: rvc render shm_queue video insert shm video failed!");
		return Error_Unexpect;
	}
	else
	{
		//DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("video render shm_queue video insert shm video ok, and nowtime is %d, and frame id is %d.", nowtime, iframeid);
		return Error_Succeed;
	}
}

static int get_local_video_frame(video_frame** frame, int itype, Clibvideoqueue* local_video_queue, IplImage* personimage, IplImage* personmask)
{
	video_frame* tmp_frame_preview = NULL;
	tmp_frame_preview = video_frame_new(REC_COMMON_VIDEO_PREVIEW_WIDTH, REC_COMMON_VIDEO_PREVIEW_HEIGHT, VIDEO_FORMAT_RGB24);
	videoq_frame frm;
	frm.data = tmp_frame_preview->data[0];
	//DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("%s:%d, session->video_shm_q_preview = 0x%08x.", __FUNCTION__, __LINE__, local_video_queue);
	bool result = local_video_queue->GetVideo(&frm, VIDEOQUEUE_FLAG_HORIZONTAL_FLIP);
	if (result){
		if (1 == itype) {
			if (NULL != personimage && NULL != personmask)
			{
				IplImage* img = cvCreateImageHeader(cvSize(frm.width, frm.height), IPL_DEPTH_8U, 3);
				img->imageData = (char*)frm.data;
				if (frm.width != personimage->width) {
					IplImage* tmp = cvCreateImage(cvSize(frm.width, frm.height), IPL_DEPTH_8U, 3);
					IplImage* tmpmask = cvCreateImage(cvSize(frm.width, frm.height), IPL_DEPTH_8U, 1);
					cvResize(personimage, tmp);
					cvResize(personmask, tmpmask);
					cvAdd(img, tmp, img, tmpmask);
					cvReleaseImage(&tmp);
					cvReleaseImage(&tmpmask);
				}
				else {
					cvAdd(img, personimage, img, personmask);
				}
				cvReleaseImageHeader(&img);
			}
		}
		*frame = tmp_frame_preview;
	}

	return 0;
}


int rvc_remote_video_render(rvc_video_render_t* prender, void* videoframe)
{
	if (NULL != prender->premote_render) {
		video_frame* echoframe = NULL;
		if (0 == translate_image_resolution(&echoframe, prender->location_param.iremote_view_cx, prender->location_param.iremote_view_cy, (const video_frame*)videoframe)) {
			prender->premote_render->RenderVideoFrame(echoframe, RVC_FLIP_VERTICAL);
			video_frame_delete(echoframe);
			echoframe = NULL;
		}
		else {
			prender->premote_render->RenderVideoFrame((video_frame*)videoframe, RVC_FLIP_VERTICAL);
		}
	}

	return 0;
}

void* rvc_videorender_func(void* arg)
{
	LOG_FUNCTION();
	rvc_video_render_t* param = (rvc_video_render_t*)arg;
	int ilocal_video_fresh_time = param->location_param.ilocal_fresh_time;
	Clibvideoqueue* local_video_queue = new Clibvideoqueue(REC_COMMON_VIDEO_ENV_SHM_PREVIEW_QUEUE);

	char strPersonPath[MAX_PATH] = { 0 };
	snprintf(strPersonPath, MAX_PATH, "./bin/rxk.jpg");
	IplImage* personimage = NULL;
	IplImage* personmask = NULL;

	if (ExistsFile(strPersonPath)){
		personimage = cvLoadImage(strPersonPath);
		personmask = cvLoadImage(strPersonPath, 0);
		DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("Load person img Success");
	}

	if (NULL != param->plocal_render) {
		videorender_param_t tparam = { 0 };
		tparam.icx = param->location_param.ilocal_view_x;
		tparam.icy = param->location_param.ilocal_view_y;
		tparam.uwidth = param->location_param.ilocal_view_cx;
		tparam.uheight = param->location_param.ilocal_view_cy;
		tparam.ivideoformat = VIDEO_FORMAT_RGB24;
		if (0 == param->plocal_render->VideoRenderSetParam(&tparam)) {
			param->plocal_render->HideVideoWindow();
		}
		else {
			DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("%s:%d set video render param failed!", __FUNCTION__, __LINE__);
			return 0;
		}
	}

	if (0 != param->location_param.iremote_view_cx && 0 != param->location_param.iremote_view_cy) {
		DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("%s:%d create remote video width = %d, height = %d.", __FUNCTION__, __LINE__, param->location_param.iremote_view_cx, param->location_param.iremote_view_cy);
		if (NULL != param->premote_render) {
			videorender_param_t tparam_remote = { 0 };
			tparam_remote.icx = param->location_param.iremote_view_x;
			tparam_remote.icy = param->location_param.iremote_view_y;
			tparam_remote.uwidth = param->location_param.iremote_view_cx;
			tparam_remote.uheight = param->location_param.iremote_view_cy;
			tparam_remote.ivideoformat = VIDEO_FORMAT_RGB24;
			if (0 == param->premote_render->VideoRenderSetParam(&tparam_remote)) {
				DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("%s:%d video render set param success.",__FUNCTION__, __LINE__);
			}
		}
		else {
			DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("remote video render is null.");
		}
	}

	if (NULL != param->plocal_render) {
		bool bshow_local = false;
		bool bset = true;

		param->plocal_render->StartVideoRender();
		for (; ; ) {
			struct timespec ts;
			clock_gettime(CLOCK_REALTIME, &ts);
			long unsec = ts.tv_nsec + (1000 * 1000 * ilocal_video_fresh_time);
			ts.tv_sec += (unsec / 1000000000);
			ts.tv_nsec = (unsec % 1000000000);
			if (0 != sem_timedwait(&param->ui_stop_sem, &ts) && (ETIMEDOUT == errno))
			{
				video_frame* local_video_frame = NULL;
				int iwindowstate = param->cb.on_window_type(param->cb.user_data);
				get_local_video_frame(&local_video_frame, iwindowstate, local_video_queue, personimage, personmask);

				if (NULL != local_video_frame) {
					video_frame* localframe = NULL;
					if (0 == translate_image_resolution(&localframe, param->location_param.ilocal_view_cx, param->location_param.ilocal_view_cy, local_video_frame)) {
						param->plocal_render->RenderVideoFrame(localframe, RVC_FLIP_VERTICAL);
						video_frame_delete(localframe);
						localframe = NULL;
					}
					else {
						param->plocal_render->RenderVideoFrame(local_video_frame, RVC_FLIP_VERTICAL);
					}

					bshow_local = true;
					video_frame_delete(local_video_frame);
					local_video_frame = NULL;
				}
				else {
					//DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("%s:%d,get video from shm preview queue failed!", __FUNCTION__, __LINE__);
				}
			}
			else {
				DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("%s:%d videorender_func exit!", __FUNCTION__, __LINE__);
				param->plocal_render->HideVideoWindow();
				break;
			}

			if (bset) {
				if (bshow_local) {
					param->plocal_render->ShowVideoWindow();
					param->premote_render->ShowVideoWindow();
					bset = false;
				}
			}
		}
		param->plocal_render->StopVideoRender();
	}

	if (NULL != personimage) {
		cvReleaseImage(&personimage);
	}

	if (NULL != personmask) {
		cvReleaseImage(&personmask);
	}

	return 0;
}


void* rvc_remote_videorender_func(void* arg)
{
	LOG_FUNCTION();

	rvc_video_render_t* param = (rvc_video_render_t*)arg;
	int iremote_video_fresh_time = param->location_param.iremote_fresh_time;
	char strImgPath[MAX_PATH] = { 0 };
	int irecord_video_frame_width = REC_COMMON_VIDEO_SNAPSHOT_PREVIEW_WIDTH;
	int irecord_video_frame_heigt = REC_COMMON_VIDEO_SNAPSHOT_PREVIEW_HEIGHT;

	Clibvideoqueue* video_shm_q_remote = new Clibvideoqueue(REC_COMMON_VIDEO_REMOTE_SHM_RTP_QUEUE);
	
	snprintf(strImgPath, MAX_PATH, "%s", "./bin/agent.jpg");

	video_frame* remote_frame = video_frame_new(irecord_video_frame_width, irecord_video_frame_heigt, VIDEO_FORMAT_RGB24);
	video_frame_fill_black(remote_frame);

	if (ExistsFile(strImgPath)){
		IplImage* img = cvLoadImage(strImgPath, 1);
		videoq_frame* vframe = new videoq_frame;

		if (NULL != img) {
			DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("load img success");
			vframe->format = VIDEOQ_FORMAT_RGB24;
			vframe->framesize = img->imageSize;
			vframe->width = img->width;
			vframe->height = img->height;
			vframe->data = new unsigned char[img->imageSize];
			memcpy(vframe->data, img->imageData, img->imageSize);
			cvReleaseImage(&img);
		}

		SwsContext* sws = sws_getContext(vframe->width, vframe->height, PIX_FMT_BGR24,
			irecord_video_frame_width,
			irecord_video_frame_heigt,
			PIX_FMT_BGR24,
			SWS_POINT, NULL, NULL, NULL);

		uint8_t* src_data[4] = { (unsigned char*)vframe->data + (vframe->height - 1) * vframe->width * 3, NULL, NULL, NULL };
		int src_linesize[4] = { -vframe->width * 3,0,0,0 };
		unsigned char* dst[4] = { remote_frame->data[0],NULL,NULL,NULL };
		int dst_linesize[4] = { remote_frame->linesize[0],0,0,0 };
		sws_scale(sws, src_data, src_linesize, 0, vframe->height, dst, dst_linesize);
		sws_freeContext(sws);

		if (vframe) {
			delete vframe->data;
			delete vframe;
			vframe = NULL;
		}
	}

	video_frame* showframe = NULL;
	if (-1 == translate_image_resolution(&showframe, param->location_param.iremote_view_cx, param->location_param.iremote_view_cy, remote_frame)) {
		showframe = video_frame_new(param->location_param.iremote_view_cx, param->location_param.iremote_view_cy, VIDEO_FORMAT_RGB24);
		video_frame_fill_black(showframe);
		video_frame_copy(showframe, remote_frame);
	}

	int iframeid = 0;
	int rc = rvc_video_shm_enqueue(video_shm_q_remote, remote_frame, VIDEOQUEUE_FLAG_VERTICAL_FLIP, iframeid++);
	if (rc != Error_Succeed) {
		DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("insert agent picture to remote video queue failed.");
	}

	for (; ; ) {
		struct timespec ts;
		clock_gettime(CLOCK_REALTIME, &ts);
		long unsec = ts.tv_nsec + (1000 * 1000 * iremote_video_fresh_time);
		ts.tv_sec += (unsec / 1000000000);
		ts.tv_nsec = (unsec % 1000000000);
		if (0 != sem_timedwait(&param->remote_render_stop_sem, &ts) && (ETIMEDOUT == errno)){
			rvc_remote_video_render(param, showframe);
			if (NULL != video_shm_q_remote){
				rc = rvc_video_shm_enqueue(video_shm_q_remote, remote_frame, VIDEOQUEUE_FLAG_VERTICAL_FLIP, iframeid++);
				if (rc != Error_Succeed){
					DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("insert agent picture to remote video queue failed.");
				}
			}
		}
		else {
			DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("%s:%d record remote video render_func exit!", __FUNCTION__, __LINE__);
			break;
		}
	}

	video_frame_delete(showframe);
	showframe = NULL;

	video_frame_delete(remote_frame);
	remote_frame = NULL;

	return 0;
}


int rvc_start_video_render(rvc_video_render_t* prender, bool bremote, rvc_video_box_move_callback_t* cb)
{
	if (0 != sem_init(&prender->ui_stop_sem, 0, 0)) {
		DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("%s:%d create stop sem failed!", __FUNCTION__, __LINE__);
		return Error_Resource;
	}
	else {
		DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("%s:%d create stop sem success!", __FUNCTION__, __LINE__);
	}

	int err = pthread_create(&prender->ui_threadid, NULL, rvc_videorender_func, prender);
	if (Error_Succeed == err) {
		DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("create video render thread success, thread id is %u.", prender->ui_threadid);
	}
	else {
		DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("create video render thread failed.");
		return Error_Resource;
	}

	if (bremote) {
		if (0 != sem_init(&prender->remote_render_stop_sem, 0, 0)) {
			DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("%s:%d create remote stop sem failed!", __FUNCTION__, __LINE__);
			return Error_Resource;
		}
		else {
			DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("%s:%d create remote stop sem success!", __FUNCTION__, __LINE__);
		}
		err = pthread_create(&prender->remote_render_threadid, NULL, rvc_remote_videorender_func, prender);
		if (Error_Succeed == err) {
			DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("create video render thread success, thread id is %u.", prender->ui_threadid);
		}
		else {
			DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("create video render thread failed.");
			return Error_Resource;
		}
	}

	return err;
}


int rvc_stop_video_render(rvc_video_render_t* prender)
{
	if (NULL == prender) {
		return Error_Param;
	}

	sem_post(&prender->ui_stop_sem);
	rvc_stop_remote_video_render(prender);
	if (prender->ui_threadid > 0) {
		if (0 == pthread_join(prender->ui_threadid, NULL)) {
			DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("video render thread %u pthread join success.", prender->ui_threadid);
			prender->ui_threadid = 0;
		}
		else {
			DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("video render thread pthread join error for %s.", strerror(errno));
		}
	}
	DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("video render thread exit!");
	return Error_Succeed;
}

int rvc_stop_remote_video_render(rvc_video_render_t* prender)
{
	if (prender->remote_render_threadid > 0) {
		sem_post(&prender->remote_render_stop_sem);
		if (0 == pthread_join(prender->remote_render_threadid, NULL)) {
			DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("remote video render thread %u pthread join success.", prender->remote_render_threadid);
			prender->remote_render_threadid = 0;
		}
		else {
			DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("remote video render thread pthread join error for %s.", strerror(errno));
		}
		DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("remote video render thread exit!");
	}

	return Error_Succeed;
}