#include "video_render.h" #include "SpBase.h" #include "Event.h" #include "videoutil.h" #include "libvideoqueue.h" #include "rvc_media_common.h" #include "fileutil.h" extern "C"{ #include #include #include } #include "video_common/ffmpeg_api_cpp_adapter.h" #include "cv.h" #include "highgui.h" #include "y2k_time.h" static int translate_image_resolution(video_frame** dstframe, const int iwidth, const int iheight, const video_frame* psrcframe) { int i = -1; if (iwidth == psrcframe->width && iheight == psrcframe->height) { return i; } video_frame* pframe = video_frame_new(iwidth, iheight, VIDEO_FORMAT_RGB24); video_frame_fill_black(pframe); SwsContext* sws = sws_getCachedContext(NULL, psrcframe->width, psrcframe->height, PIX_FMT_BGR24, iwidth, iheight, PIX_FMT_BGR24, SWS_LANCZOS, NULL, NULL, NULL); sws_scale(sws, psrcframe->data, psrcframe->linesize, 0, psrcframe->height, pframe->data, pframe->linesize); sws_freeContext(sws); *dstframe = pframe; i = 0; return i; } static int rvc_video_shm_enqueue(Clibvideoqueue* shm_queue, video_frame* frame, int flags, int iframeid) { videoq_frame tmp_frm; tmp_frm.data = frame->data[0]; tmp_frm.framesize = frame->width * frame->height * 3; tmp_frm.format = VIDEOQ_FORMAT_RGB24; tmp_frm.width = frame->width; tmp_frm.height = frame->height; tmp_frm.iframeid = iframeid; unsigned int nowtime = y2k_time_now(); if (!shm_queue->InsertVideo(&tmp_frm, flags, nowtime)){ DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("caution: rvc render shm_queue video insert shm video failed!"); return Error_Unexpect; } else { //DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("video render shm_queue video insert shm video ok, and nowtime is %d, and frame id is %d.", nowtime, iframeid); return Error_Succeed; } } static int get_local_video_frame(video_frame** frame, int itype, Clibvideoqueue* local_video_queue, IplImage* personimage, IplImage* personmask) { video_frame* tmp_frame_preview = NULL; tmp_frame_preview = video_frame_new(REC_COMMON_VIDEO_PREVIEW_WIDTH, REC_COMMON_VIDEO_PREVIEW_HEIGHT, VIDEO_FORMAT_RGB24); videoq_frame frm; frm.data = tmp_frame_preview->data[0]; //DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("%s:%d, session->video_shm_q_preview = 0x%08x.", __FUNCTION__, __LINE__, local_video_queue); bool result = local_video_queue->GetVideo(&frm, VIDEOQUEUE_FLAG_HORIZONTAL_FLIP); if (result){ if (1 == itype) { if (NULL != personimage && NULL != personmask) { IplImage* img = cvCreateImageHeader(cvSize(frm.width, frm.height), IPL_DEPTH_8U, 3); img->imageData = (char*)frm.data; if (frm.width != personimage->width) { IplImage* tmp = cvCreateImage(cvSize(frm.width, frm.height), IPL_DEPTH_8U, 3); IplImage* tmpmask = cvCreateImage(cvSize(frm.width, frm.height), IPL_DEPTH_8U, 1); cvResize(personimage, tmp); cvResize(personmask, tmpmask); cvAdd(img, tmp, img, tmpmask); cvReleaseImage(&tmp); cvReleaseImage(&tmpmask); } else { cvAdd(img, personimage, img, personmask); } cvReleaseImageHeader(&img); } } *frame = tmp_frame_preview; } return 0; } int rvc_remote_video_render(rvc_video_render_t* prender, void* videoframe) { if (NULL != prender->premote_render) { video_frame* echoframe = NULL; if (0 == translate_image_resolution(&echoframe, prender->location_param.iremote_view_cx, prender->location_param.iremote_view_cy, (const video_frame*)videoframe)) { prender->premote_render->RenderVideoFrame(echoframe, RVC_FLIP_VERTICAL); video_frame_delete(echoframe); echoframe = NULL; } else { prender->premote_render->RenderVideoFrame((video_frame*)videoframe, RVC_FLIP_VERTICAL); } } return 0; } void* rvc_videorender_func(void* arg) { rvc_video_render_t* param = (rvc_video_render_t*)arg; int ilocal_video_fresh_time = param->location_param.ilocal_fresh_time; Clibvideoqueue* local_video_queue = new Clibvideoqueue(REC_COMMON_VIDEO_ENV_SHM_PREVIEW_QUEUE); char strPersonPath[MAX_PATH] = { 0 }; snprintf(strPersonPath, MAX_PATH, "./bin/rxk.jpg"); IplImage* personimage = NULL; IplImage* personmask = NULL; if (ExistsFile(strPersonPath)){ personimage = cvLoadImage(strPersonPath); personmask = cvLoadImage(strPersonPath, 0); } if (NULL != param->plocal_render) { videorender_param_t tparam = { 0 }; tparam.icx = param->location_param.ilocal_view_x; tparam.icy = param->location_param.ilocal_view_y; tparam.uwidth = param->location_param.ilocal_view_cx; tparam.uheight = param->location_param.ilocal_view_cy; tparam.ivideoformat = VIDEO_FORMAT_RGB24; if (0 == param->plocal_render->VideoRenderSetParam(&tparam)) { param->plocal_render->HideVideoWindow(); } else { return 0; } } if (0 != param->location_param.iremote_view_cx && 0 != param->location_param.iremote_view_cy) { if (NULL != param->premote_render) { videorender_param_t tparam_remote = { 0 }; tparam_remote.icx = param->location_param.iremote_view_x; tparam_remote.icy = param->location_param.iremote_view_y; tparam_remote.uwidth = param->location_param.iremote_view_cx; tparam_remote.uheight = param->location_param.iremote_view_cy; tparam_remote.ivideoformat = VIDEO_FORMAT_RGB24; if (0 != param->premote_render->VideoRenderSetParam(&tparam_remote)) { DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("video render set param failed."); } } else { DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("remote video render is null."); } } if (NULL != param->plocal_render) { bool bshow_local = false; bool bset = true; param->plocal_render->StartVideoRender(); for (; ; ) { struct timespec ts; clock_gettime(CLOCK_REALTIME, &ts); long unsec = ts.tv_nsec + (1000 * 1000 * ilocal_video_fresh_time); ts.tv_sec += (unsec / 1000000000); ts.tv_nsec = (unsec % 1000000000); if (0 != sem_timedwait(¶m->ui_stop_sem, &ts) && (ETIMEDOUT == errno)) { video_frame* local_video_frame = NULL; int iwindowstate = param->cb.on_window_type(param->cb.user_data); get_local_video_frame(&local_video_frame, iwindowstate, local_video_queue, personimage, personmask); if (NULL != local_video_frame) { video_frame* localframe = NULL; if (0 == translate_image_resolution(&localframe, param->location_param.ilocal_view_cx, param->location_param.ilocal_view_cy, local_video_frame)) { param->plocal_render->RenderVideoFrame(localframe, RVC_FLIP_VERTICAL); video_frame_delete(localframe); localframe = NULL; } else { param->plocal_render->RenderVideoFrame(local_video_frame, RVC_FLIP_VERTICAL); } bshow_local = true; video_frame_delete(local_video_frame); local_video_frame = NULL; } } else { param->plocal_render->HideVideoWindow(); break; } if (bset) { if (bshow_local) { param->plocal_render->ShowVideoWindow(); param->premote_render->ShowVideoWindow(); bset = false; } } } param->plocal_render->StopVideoRender(); } if (NULL != personimage) { cvReleaseImage(&personimage); } if (NULL != personmask) { cvReleaseImage(&personmask); } return 0; } void* rvc_remote_videorender_func(void* arg) { LOG_FUNCTION(); rvc_video_render_t* param = (rvc_video_render_t*)arg; int iremote_video_fresh_time = param->location_param.iremote_fresh_time; char strImgPath[MAX_PATH] = { 0 }; int irecord_video_frame_width = REC_COMMON_VIDEO_SNAPSHOT_PREVIEW_WIDTH; int irecord_video_frame_heigt = REC_COMMON_VIDEO_SNAPSHOT_PREVIEW_HEIGHT; Clibvideoqueue* video_shm_q_remote = new Clibvideoqueue(REC_COMMON_VIDEO_REMOTE_SHM_RTP_QUEUE); snprintf(strImgPath, MAX_PATH, "%s", "./bin/agent.jpg"); video_frame* remote_frame = video_frame_new(irecord_video_frame_width, irecord_video_frame_heigt, VIDEO_FORMAT_RGB24); video_frame_fill_black(remote_frame); if (ExistsFile(strImgPath)){ IplImage* img = cvLoadImage(strImgPath, 1); videoq_frame* vframe = new videoq_frame; if (NULL != img) { vframe->format = VIDEOQ_FORMAT_RGB24; vframe->framesize = img->imageSize; vframe->width = img->width; vframe->height = img->height; vframe->data = new unsigned char[img->imageSize]; memcpy(vframe->data, img->imageData, img->imageSize); cvReleaseImage(&img); } SwsContext* sws = sws_getContext(vframe->width, vframe->height, PIX_FMT_BGR24, irecord_video_frame_width, irecord_video_frame_heigt, PIX_FMT_BGR24, SWS_POINT, NULL, NULL, NULL); uint8_t* src_data[4] = { (unsigned char*)vframe->data + (vframe->height - 1) * vframe->width * 3, NULL, NULL, NULL }; int src_linesize[4] = { -vframe->width * 3,0,0,0 }; unsigned char* dst[4] = { remote_frame->data[0],NULL,NULL,NULL }; int dst_linesize[4] = { remote_frame->linesize[0],0,0,0 }; sws_scale(sws, src_data, src_linesize, 0, vframe->height, dst, dst_linesize); sws_freeContext(sws); if (vframe) { delete vframe->data; delete vframe; vframe = NULL; } } video_frame* showframe = NULL; if (-1 == translate_image_resolution(&showframe, param->location_param.iremote_view_cx, param->location_param.iremote_view_cy, remote_frame)) { showframe = video_frame_new(param->location_param.iremote_view_cx, param->location_param.iremote_view_cy, VIDEO_FORMAT_RGB24); video_frame_fill_black(showframe); video_frame_copy(showframe, remote_frame); } int iframeid = 0; int rc = rvc_video_shm_enqueue(video_shm_q_remote, remote_frame, VIDEOQUEUE_FLAG_VERTICAL_FLIP, iframeid++); if (rc != Error_Succeed) { DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("insert agent picture to remote video queue failed."); } for (; ; ) { struct timespec ts; clock_gettime(CLOCK_REALTIME, &ts); long unsec = ts.tv_nsec + (1000 * 1000 * iremote_video_fresh_time); ts.tv_sec += (unsec / 1000000000); ts.tv_nsec = (unsec % 1000000000); if (0 != sem_timedwait(¶m->remote_render_stop_sem, &ts) && (ETIMEDOUT == errno)){ rvc_remote_video_render(param, showframe); if (NULL != video_shm_q_remote){ rc = rvc_video_shm_enqueue(video_shm_q_remote, remote_frame, VIDEOQUEUE_FLAG_VERTICAL_FLIP, iframeid++); if (rc != Error_Succeed){ DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("insert agent picture to remote video queue failed."); } } } else { break; } } video_frame_delete(showframe); showframe = NULL; video_frame_delete(remote_frame); remote_frame = NULL; return 0; } int rvc_start_video_render(rvc_video_render_t* prender, bool bremote, rvc_video_box_move_callback_t* cb) { if (0 != sem_init(&prender->ui_stop_sem, 0, 0)) { return Error_Resource; } int err = pthread_create(&prender->ui_threadid, NULL, rvc_videorender_func, prender); if (Error_Succeed != err) { DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("create video render thread failed."); return Error_Resource; } if (bremote) { if (0 != sem_init(&prender->remote_render_stop_sem, 0, 0)) { DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("create remote stop sem failed!"); return Error_Resource; } err = pthread_create(&prender->remote_render_threadid, NULL, rvc_remote_videorender_func, prender); if (Error_Succeed != err) { DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("create video render thread failed."); return Error_Resource; } } return err; } int rvc_stop_video_render(rvc_video_render_t* prender) { if (NULL == prender) { return Error_Param; } sem_post(&prender->ui_stop_sem); rvc_stop_remote_video_render(prender); if (prender->ui_threadid > 0) { if (0 == pthread_join(prender->ui_threadid, NULL)) { prender->ui_threadid = 0; } else { DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("video render thread pthread join error for %s.", strerror(errno)); } } return Error_Succeed; } int rvc_stop_remote_video_render(rvc_video_render_t* prender) { if (prender->remote_render_threadid > 0) { sem_post(&prender->remote_render_stop_sem); if (0 == pthread_join(prender->remote_render_threadid, NULL)) { prender->remote_render_threadid = 0; } else { DbgWithLink(LOG_LEVEL_INFO, LOG_TYPE_SYSTEM)("remote video render thread pthread join error for %s.", strerror(errno)); } } return Error_Succeed; }