#include "precompile.h" #include "videortp.h" #include "videocommon.h" #include "bs.h" #ifdef RVC_OS_WIN #define WIN32_LEAN_AND_MEAN #include #else #include #endif // RVC_OS_WIN #include #include #include #include "list.h" #include "rtp.h" #include "rtpsession.h" #include "memutil.h" #include "../rvcmediacommon/rvc_media_common.h" #include "x264.h" #include "video_common/ffmpeg_api_adapter.h" #include "video_coding/video_encoder_api.h" #include "video_coding/video_decoder.h" #include "video_coding/base/video_coding_clock.h" #include "video_coding/base/video_coding_log.h" #include "congestion_control/sim_transport/sim_external.h" #include "adaptive_jitter_buffer/video_jbuff_api.h" #include "adaptive_jitter_buffer/video_jbuff_log.h" #include "rtp_header_extension/rtp_header_extension_api.h" #include "h264_packetizer/h264_packetizer.h" #include "video_statics/video_stats.h" #include "video_common/videodebugfile.h" //接收缓冲区要大于MAX_MTU_SIZE，还要存储rtp头、udp头等 #define MAX_RECV_BUFFER_SIZE 1500 #define MAX_PLAYLOAD_SIZE 1400 #define MAX_ASSEMBLE_EXTRA_SIZE 128 #define FRAME_WIN_SIZE 20 #define RTP_SEQ_GT(x, y, n) ((x > y && x-y<=n) || (y+nconfig.dbg) { (*rtp->config.dbg)(rtp->config.user_data, fmt, arg); } else { n = vsnprintf(NULL, 0, fmt, arg); if (n > 0) { char *buf = (char*)_alloca((size_t)(n+3)); vsprintf(buf, fmt, arg); strcat(buf, "\r\n"); #ifdef _WIN32 OutputDebugStringA((LPCSTR)buf); #endif printf(buf); } } va_end(arg); } static void rtpLogevent(videortp_t* rtp, int itype, const char* strmessage) { if (rtp->config.logevent) { (*rtp->config.logevent)(rtp->config.user_data, itype, strmessage); } } //#define Dbg(__fmt__, ...) __Dbg(vrtp, __fmt__, __VA_ARGS__) static void get_start_config_bitrate(videortp_config_t *config, int *max_bitrate, int *target_bitrate, int *min_bitrate){ *max_bitrate = config->bit_rate; //bps *min_bitrate = 50*1000 > config->bit_rate ? config->bit_rate : 50*1000; //bps *target_bitrate = *max_bitrate; //bps *target_bitrate = (*target_bitrate < *min_bitrate) ? *min_bitrate : *target_bitrate; //bps } static int get_mtu(int config_mtu){ int mtu = config_mtu > MAX_PLAYLOAD_SIZE ? MAX_PLAYLOAD_SIZE : config_mtu; return mtu; } //interval 单位为ms static unsigned convert_to_rtp_timestamp_increment(int64_t interval) { // 时间戳增量 unsigned timestampIncrement = (VIDEO_CLOCK/1000*interval); return timestampIncrement; } static void yuv_buffers_combine_to_one(const video_frame *vframe, unsigned char *buffer) { int i =0, j =0, k =0; for(i = 0; i < vframe->height; i++) { memcpy((void *)(buffer+vframe->width*i), (const void *)(vframe->data[0]+vframe->linesize[0]*i), (size_t)(vframe->width)); } for(j = 0; j < vframe->height/2; j++) { memcpy((void *)(buffer+vframe->width*i+vframe->width/2*j), (const void *)(vframe->data[1]+vframe->linesize[1]*j), (size_t)(vframe->width/2)); } for(k =0; k < vframe->height/2; k++) { memcpy((void *)(buffer+vframe->width*i+vframe->width/2*j+vframe->width/2*k), (const void *)(vframe->data[2]+vframe->linesize[2]*k), (size_t)(vframe->width/2)); } } static rtpframe_part_t *rtpframe_part_create(videortp_t *vrtp) { rtpframe_part_t *part; if (!list_empty((const struct list_head *)(&vrtp->free_part_list))) { part = list_first_entry(&vrtp->free_part_list, rtpframe_part_t, entry); list_del((struct list_head *)(&part->entry)); } else { part = MALLOC_T(rtpframe_part_t); } return part; } static void rtpframe_part_destroy(videortp_t *vrtp, rtpframe_part_t *part) { list_add((struct list_head *)(&part->entry), (struct list_head *)(&vrtp->free_part_list)); } rtpframe_part_t *rtpframe_part_dupb(rtpframe_part_t *mp){ rtpframe_part_t *part = MALLOC_T(rtpframe_part_t); if (part){ part->entry = mp->entry; memcpy((void *)(part->buf), (const void *)(mp->buf), strlen((const char *)(mp->buf))); part->offset = mp->offset; part->actual_len = mp->actual_len; part->mb_in_slice = mp->mb_in_slice; part->seq = mp->seq; part->mark = mp->mark; part->psc = mp->psc; } return part; } static void rtpframe_clear(videortp_t *vrtp, rtpframe_t *frame) { if (frame->used) { list_splice_init((struct list_head *)(&frame->part_list), (struct list_head *)(&vrtp->free_part_list)); frame->used = 0; } } static unsigned short rtpframe_get_last_seq(rtpframe_t *frame) { if (!list_empty((const struct list_head *)(&frame->part_list))) { rtpframe_part_t *part = list_last_entry(&frame->part_list, rtpframe_part_t, entry); return part->seq; } else { assert(0); return 0; } } static unsigned short rtpframe_get_first_seq(rtpframe_t *frame) { if (!list_empty((const struct list_head *)(&frame->part_list))) { rtpframe_part_t *part = list_first_entry(&frame->part_list, rtpframe_part_t, entry); return part->seq; } else { assert(0); return 0; } } static int rtpframe_is_empty(rtpframe_t *frame) { if (list_empty((const struct list_head *)(&frame->part_list))) { return 1; } return 0; } static __inline void receiver_notify_peer_fast_update(videortp_t *vrtp) { DWORD dwNow = GetTickCount(); if (dwNow - vrtp->last_peer_update_tick >= PEER_UPDATE_INTERVAL) { rtpDbg(vrtp,"%I64d receiver_notify_peer_fast_update!", TimeInMilliseconds()); rtp_session_send_rtcp_h261_fir(vrtp->rtp_sess); vrtp->last_peer_update_tick = dwNow; } } static __inline void receiver_notify_peer_fast_update_now(videortp_t *vrtp) { DWORD dwNow = GetTickCount(); //rtpDbg(vrtp,"%I64d receiver_notify_peer_fast_update_now!", TimeInMilliseconds()); rtp_session_send_rtcp_h261_fir(vrtp->rtp_sess); vrtp->last_peer_update_tick = dwNow; } static __inline void receiver_force_key_frame(videortp_t *vrtp) { rtpDbg(vrtp,"%I64d receiver_force_key_frame!", TimeInMilliseconds()); #ifdef _WIN32 InterlockedCompareExchange(&vrtp->force_key_frames, 1, 0); #else LONG* plTemp = &vrtp->force_key_frames; if (0 == *plTemp){ *plTemp = 1; rtpDbg(vrtp,"%s:%d force_key_frames = %lu", __FUNCTION__, __LINE__, vrtp->force_key_frames); } #endif // _WIN32 } static void receiver_clear_window(videortp_t *vrtp) { int i; for (i = 0; i < FRAME_WIN_SIZE; ++i) { rtpframe_t *frame = &vrtp->frame_win[i]; rtpframe_clear(vrtp, frame); } vrtp->frame_duration = 0; } int receiver_msgdsize(const rtpframe_part_t *mp) { int msgsize = 0; { msgsize += (int) mp->actual_len; } return msgsize; } int video_coding_on_log(void *userdata, int level, const char* file, int line, const char* fmt, va_list vl){ videortp_t *vrtp = (videortp_t *)userdata; int n; n = vsnprintf(NULL, 0, fmt, vl); if ((DEBUG_ENCODER || DEBUG_DECODER) && n > 0 && vrtp && level >= VIDEO_CODING_DEBUG_INFO) { char *buf = (char*)_alloca((size_t)(n+3)); vsprintf(buf, fmt, vl); rtpDbg(vrtp,"%s.", buf); } return 0; } static int decoder_process(videortp_t *vrtp, uint8_t *input_buf, int input_len){ int64_t before_decode_ms; int64_t after_decode_ms; int64_t before_render_ms; int64_t after_render_ms; //decode while (input_len > 0) { video_frame *decodered_frame = NULL; int key_frame = 0; video_debug_write_play_file(AUDIO_DEC_IN, input_buf, input_len); if(DEBUG_DECODER){ rtpDbg(vrtp,"%s:%d: receiver_do_decode before decode data_len:%d.", __FILE__, __LINE__, input_len); } before_decode_ms = TimeInMilliseconds(); decodered_frame = video_decoder_process(vrtp->decoder, input_buf, input_len, &input_len, &key_frame); if(DEBUG_DECODER){ rtpDbg(vrtp,"%s:%d: receiver_do_decode after decode data_len:%d.", __FILE__, __LINE__, input_len); } after_decode_ms = TimeInMilliseconds(); if (decodered_frame == NULL){ rtpDbg(vrtp,"receiver_do_decode decode failed."); video_stats_receiver_on_decoded_frame(after_decode_ms, after_decode_ms - before_decode_ms, 0); return 0; } vrtp->prev_dec_ms = after_decode_ms; video_stats_receiver_on_decoded_frame(after_decode_ms, after_decode_ms - before_decode_ms, key_frame); //debug render file if (vrtp->config.rx_height) { if (VIDEO_FORMAT_RGB24 != decodered_frame->format) { unsigned char* video_debug_buf; int length = vrtp->config.rx_height * vrtp->config.rx_width * 3 / 2; video_debug_buf = (unsigned char*)malloc(length); yuv_buffers_combine_to_one(decodered_frame, video_debug_buf); video_debug_write_play_file(VIDEO_RENDER_IN, video_debug_buf, length); free(video_debug_buf); } } //notify render before_render_ms = TimeInMilliseconds(); if(vrtp->config.on_rx_frame){ int iret = vrtp->config.on_rx_frame(decodered_frame, vrtp->config.user_data); if (1 == iret){ #ifdef _WIN32 #else video_frame_delete(decodered_frame); decodered_frame = NULL; #endif } } else { //render is null, we need delete self. video_frame_delete(decodered_frame); decodered_frame = NULL; } after_render_ms = TimeInMilliseconds(); video_stats_receiver_on_rendered_frame(after_render_ms, after_render_ms - before_render_ms); } return input_len == 0; } void decoder_create(videortp_t *vrtp, const videortp_config_t *config){ VideoCodingLogCallBack log_func; DecoderConfig decoder_config; decoder_config.color_space = VIDEO_FORMAT_RGB24; decoder_config.decode_id = CODEC_ID_H264; decoder_config.width = config->rx_width; decoder_config.height = config->rx_height; log_func.log_fn = &video_coding_on_log; log_func.userdata = vrtp; vrtp->decoder = video_decoder_open(&decoder_config, &log_func); rtpDbg(vrtp,"decoder_create color_space: %d decode_id: %d width: %d height: %d.", decoder_config.color_space, decoder_config.decode_id, decoder_config.width, decoder_config.height); } void decoder_destroy(videortp_t *vrtp){ if (vrtp->decoder != NULL) { video_decoder_close(vrtp->decoder); } rtpDbg(vrtp,"decoder_destroy."); } /******************************standard_h264接收处理 start***************************/ static void receiver_standard_h264_update_sps(videortp_t *d, rtpframe_part_t *sps){ if (d->sps){ free((void *)(d->sps)); } d->sps = rtpframe_part_dupb(sps); } static void receiver_standard_h264_update_pps(videortp_t *d, rtpframe_part_t *pps){ if (d->pps){ free((void *)(d->pps)); } if (pps){ d->pps=rtpframe_part_dupb(pps); } else{ d->pps=NULL; } } static bool_t receiver_standard_h264_check_sps_pps_change(videortp_t *d, rtpframe_part_t *sps, rtpframe_part_t *pps){ bool_t ret1=FALSE,ret2=FALSE; if (d->sps){ if (sps){ ret1=(receiver_msgdsize(sps)!=receiver_msgdsize(d->sps)) || (memcmp((const void *)(d->sps->buf),(const void *)(sps->buf),(size_t)receiver_msgdsize(sps))!=0); if (ret1) { receiver_standard_h264_update_sps(d,sps); receiver_standard_h264_update_pps(d,NULL); } } }else if (sps) { receiver_standard_h264_update_sps(d,sps); } if (d->pps){ if (pps){ ret2=(receiver_msgdsize(pps)!=receiver_msgdsize(d->pps)) || (memcmp((const void *)(d->pps->buf), (const void *)(pps->buf), (size_t)receiver_msgdsize(pps))!=0); if (ret2) { receiver_standard_h264_update_pps(d,pps); } } }else if (pps) { receiver_standard_h264_update_pps(d,pps); } return ret1 || ret2; } static int receiver_standard_h264_add_startcode(videortp_t *vrtp, rtpframe_t *frame, uint8_t *output_buf, bool_t *new_sps_pps) { rtpframe_part_t *pos = NULL; uint8_t *dst = NULL; bool_t start_picture=TRUE; int output_len = 0; dst = output_buf; list_for_each_entry(pos, &frame->part_list, rtpframe_part_t, entry) { x264_hdr *hdr = (x264_hdr*)&pos->buf[pos->offset]; unsigned char* src = (unsigned char*)&pos->buf[pos->offset]; uint8_t nalu_type = hdr->id; if (nalu_type==NAL_SPS){ *new_sps_pps=receiver_standard_h264_check_sps_pps_change(vrtp,pos,NULL) || *new_sps_pps; } if (nalu_type==NAL_PPS){ *new_sps_pps=receiver_standard_h264_check_sps_pps_change(vrtp,NULL,pos) || *new_sps_pps; } if (start_picture || nalu_type==NAL_SPS || nalu_type==NAL_PPS ){ *dst++=0; start_picture=FALSE; } *dst++ = 0; *dst++ = 0; *dst++ = 1; *dst++=*src++; while(src < (unsigned char*)&(pos->buf[pos->offset]) + pos->actual_len - 3){ if (src[0]==0 && src[1]==0 && src[2]<3){ *dst++=0; *dst++=0; *dst++=3; src+=2; } *dst++=*src++; } *dst++=*src++; *dst++=*src++; *dst++=*src++; } output_len = dst - output_buf; return output_len; } static int receiver_standard_h264_add_encoded_buffer_startcode(videortp_t *vrtp, EncodedImage *encoded_image, uint8_t *output_buf) { uint8_t *dst = NULL; bool_t start_picture=TRUE; int output_len = 0; int i = 0; dst = output_buf; for (i = 0; i < encoded_image->fragmentation_header_.fragmentationVectorSize; i++) { x264_hdr *hdr = (x264_hdr*)&encoded_image->encoded_data_[encoded_image->fragmentation_header_.fragmentationOffset[i]]; unsigned char* src = (unsigned char*)&encoded_image->encoded_data_[encoded_image->fragmentation_header_.fragmentationOffset[i]]; uint8_t nalu_type = hdr->id; if (start_picture || nalu_type==NAL_SPS || nalu_type==NAL_PPS){ *dst++=0; start_picture=FALSE; } *dst++ = 0; *dst++ = 0; *dst++ = 1; *dst++=*src++; while(src < (unsigned char*)&encoded_image->encoded_data_[encoded_image->fragmentation_header_.fragmentationOffset[i]] + encoded_image->fragmentation_header_.fragmentationLength[i] - 3){ if (src[0]==0 && src[1]==0 && src[2]<3){ *dst++=0; *dst++=0; *dst++=3; src+=2; } *dst++=*src++; } *dst++=*src++; *dst++=*src++; *dst++=*src++; } output_len = dst - output_buf; return output_len; } static int receiver_standard_h264_calculate_mb_in_slice(char* psrc, size_t ulen) { int ret = -1; bs_t bs_data; if (NULL == psrc || 0 == ulen){ return ret; } bs_init(&bs_data, psrc, ulen); ret = bs_read_ue(&bs_data); return ret; } static int receiver_standard_h264_append_rtpframe_part(videortp_t *rtp, rtpframe_t *frame, unsigned ts, unsigned key, rtpframe_part_t *new_part) { if (frame->used && frame->ts != ts) { rtpframe_clear(rtp, frame); } if (frame->used == 0){ frame->ts = ts; frame->key = key; frame->cur_max_mb = 0; } if (frame->cur_max_mb <= new_part->mb_in_slice) { frame->cur_max_mb = new_part->mb_in_slice; list_add_tail(&new_part->entry, &frame->part_list); frame->used++; } else { rtpframe_part_t *pos; unsigned short temp_seq; list_for_each_entry(pos, &frame->part_list, rtpframe_part_t, entry) { if (pos->mb_in_slice > new_part->mb_in_slice) { break; } } temp_seq = new_part->seq; new_part->seq = pos->seq; pos->seq = temp_seq; __list_add(&new_part->entry, pos->entry.prev, &pos->entry); frame->used++; } return 1; } //ts不同, 一帧结束，返回此帧 static int receiver_standard_h264_find_frame(videortp_t *rtp, rtpframe_part_t* part_data, unsigned mark, unsigned ts, unsigned key) { int i = rtp->frame_win_start; if (rtp->frame_win[i].ts == ts) { receiver_standard_h264_append_rtpframe_part(rtp, &rtp->frame_win[i], ts, key, part_data); return -1; } else{ int index = (rtp->frame_win_start + 1)%FRAME_WIN_SIZE; rtpframe_t *frame = &rtp->frame_win[i]; receiver_standard_h264_append_rtpframe_part(rtp, &rtp->frame_win[index], ts, key, part_data); rtp->frame_win_start = index; return i; } } static int receiver_standard_h264_on_jbuffer_request_key_frame(void *userdata){ videortp_t *vrtp = (videortp_t *)userdata; if (vrtp) { receiver_notify_peer_fast_update_now((vrtp)); } } static int receiver_h264_on_receive_encoded_image(EncodedImage *encoded_image, unsigned int timestamp, unsigned short first_seq, unsigned short last_seq, void *userdata){ videortp_t *vrtp = (videortp_t *)userdata; if (vrtp) { uint8_t *input_buf = NULL; int input_len = 0; int i = 0; int decoded; if(DEBUG_RX){ rtpDbg(vrtp,"%s:%d: receiver_h264_on_receive_encoded_image timestamp:%u first_seq:%u last_seq:%u key: %d len: %d.", __FILE__, __LINE__, timestamp, first_seq, last_seq, encoded_image->key_frame, encoded_image->size_); rtpDbg(vrtp,"%s:%d: receiver_h264_on_receive_encoded_image fragmentationSize: %d.", __FILE__, __LINE__, encoded_image->fragmentation_header_.fragmentationVectorSize); for (i = 0;i < encoded_image->fragmentation_header_.fragmentationVectorSize;++i){ rtpDbg(vrtp,"%s:%d: receiver_h264_on_receive_encoded_image index: %d fragmentationLength: %d fragmentationOffset: %d.", __FILE__, __LINE__, i, encoded_image->fragmentation_header_.fragmentationLength[i], encoded_image->fragmentation_header_.fragmentationOffset[i]); } } if (encoded_image->key_frame || vrtp->prev_dec_seq+1 == first_seq) { input_len = encoded_image->size_ + MAX_ASSEMBLE_EXTRA_SIZE; input_buf = (uint8_t*)_alloca(input_len); memset(input_buf, 0, input_len); //00->003,同时加上startcode if (encoded_image->pt_ == REC_COMMON_VIDEO_PT){ memcpy(input_buf, encoded_image->encoded_data_, encoded_image->size_); input_len = encoded_image->size_; } else { if (vrtp->h264_packetizer){ int status; unsigned int whole_len = 0; for (i=0; ifragmentation_header_.fragmentationVectorSize; ++i) { if (whole_len + encoded_image->fragmentation_header_.fragmentationLength[i] > input_len) { return 0; } status = h264_unpacketize( vrtp->h264_packetizer, &encoded_image->encoded_data_[encoded_image->fragmentation_header_.fragmentationOffset[i]], encoded_image->fragmentation_header_.fragmentationLength[i], input_buf, input_len, &whole_len); if (status != 0) { continue; } } input_len = whole_len; }else { input_len = receiver_standard_h264_add_encoded_buffer_startcode(vrtp, encoded_image, input_buf); } } if(DEBUG_RX){ rtpDbg(vrtp,"%s:%d: receiver_h264_on_receive_encoded_image encoded_size:%d frame_len:%d now:%I64d.", __FILE__, __LINE__, encoded_image->size_, input_len, TimeInMilliseconds()); } //decode decoded = decoder_process(vrtp, input_buf, input_len); if (decoded) { vrtp->prev_dec_seq = last_seq; vrtp->prev_dec_ts = timestamp; } else { rtpDbg(vrtp,"receiver_h264_on_receive_encoded_image decode_frame failed!"); } } } return 0; } static int receiver_standard_h264_decode(videortp_t *vrtp, rtpframe_t *frame) { if (frame->used) { rtpframe_part_t *pos = NULL; int got_picture = 0; uint8_t *input_buf = NULL; int input_len = 0; input_buf = (uint8_t*)_alloca(frame->used * MAX_RECV_BUFFER_SIZE + FF_INPUT_BUFFER_PADDING_SIZE); memset(input_buf, 0, frame->used * MAX_RECV_BUFFER_SIZE + FF_INPUT_BUFFER_PADDING_SIZE); //00->003,同时加上startcode { bool_t need_reinit=FALSE; input_len = receiver_standard_h264_add_startcode(vrtp, frame, input_buf, &need_reinit); } if(DEBUG_RX){ rtpDbg(vrtp,"%s:%d: receiver_standard_h264_decode frame_used:%d frame_len:%d now:%I64d.", __FILE__, __LINE__, frame->used, input_len, TimeInMilliseconds()); } //decode return decoder_process(vrtp, input_buf, input_len); } return 0; } rtpframe_t *receiver_standard_h264_assemble(videortp_t *vrtp, rtpframe_part_t *part_data, int len, unsigned short seq, unsigned mark, unsigned ts){ int ret = -1; int iret = -1; rtpframe_t *frame = NULL; int key = 0; x264_hdr *hdr = (x264_hdr*)&part_data->buf[0]; uint8_t nalu_type = hdr->id; if(NAL_SPS == nalu_type || NAL_PPS == nalu_type){ part_data->mb_in_slice = 0; key = 1; }else{ part_data->mb_in_slice = receiver_standard_h264_calculate_mb_in_slice(&part_data->buf[1], part_data->actual_len - 1); } if (NAL_SLICE_IDR == nalu_type){ key = 1; } part_data->actual_len = len; part_data->offset = 0; part_data->mark = mark; part_data->seq = seq; if(DEBUG_RX){ rtpDbg(vrtp,"%s:%d: receiver_standard_h264_assemble actual_len:%d seq:%d key:%d.", __FILE__, __LINE__, len, seq, key); } iret = receiver_standard_h264_find_frame(vrtp, part_data, mark, ts, key); if (iret != -1) { return &vrtp->frame_win[iret]; } return NULL; } static int receiver_process_standard_h264_packet( videortp_t *vrtp, rtpframe_part_t *part_data, int len, unsigned short seq, unsigned mark, unsigned ts, unsigned pt) { #if 1 VideoJBufferPacketInfo packet_info = {0}; x264_hdr *hdr = (x264_hdr*)&part_data->buf[part_data->offset]; uint8_t nalu_type = NAL_UNKNOWN; int actual_len = part_data->actual_len; int offset = part_data->offset; packet_info.length = actual_len; packet_info.packet_type = NORMAL_PACKET_IN_FRAME; if(mark){ packet_info.packet_type = LAST_PACKET_IN_FRAME; } packet_info.seq = seq; packet_info.timestamp = ts; nalu_type = hdr->id; if (vrtp->h264_packetizer) { h264_unpacketize_get_original_nal_type(vrtp->h264_packetizer, &part_data->buf[offset], actual_len, &nalu_type); } if(NAL_SPS == nalu_type || NAL_PPS == nalu_type || NAL_SLICE_IDR == nalu_type){ packet_info.is_key = 1; } packet_info.pt = pt; if(DEBUG_RX){ rtpDbg(vrtp,"%s:%d: receiver_process_standard_h264_packet len:%d seq:%d mark:%d ts:%d nalu_type:%u offset:%d actual_len:%d.", __FILE__, __LINE__, len, seq, mark, ts, nalu_type, offset, actual_len); } video_jbuffer_api_receive_packet(vrtp->jbuffer, &part_data->buf[offset], &packet_info); rtpframe_part_destroy(vrtp, part_data); #else rtpframe_t *frame = NULL; frame = receiver_standard_h264_assemble(vrtp, part_data, len, seq, mark, ts); if (frame != NULL) { int decoded = 0; //取出的frame可能为空(在第一次收数据时),需要先判空 if (!rtpframe_is_empty(frame) && (frame->key || vrtp->prev_dec_seq+1 == rtpframe_get_first_seq(frame))){ decoded = receiver_standard_h264_decode(vrtp, frame); if (decoded) { //Dbg("decode_frame ok!"); if (vrtp->frame_duration==0 && vrtp->prev_dec_seq+1 == rtpframe_get_first_seq(frame)) { vrtp->frame_duration = frame->ts - vrtp->prev_dec_ts; rtpDbg(vrtp,"decode frame duration: %d", vrtp->frame_duration); } vrtp->prev_dec_seq = rtpframe_get_last_seq(frame); vrtp->prev_dec_ts = frame->ts; } else { rtpDbg(vrtp,"decode_frame failed!"); } } rtpframe_clear(vrtp, frame); } #endif return 0; } /******************************standard_h264接收处理 end***************************/ #if 0 #endif /******************************custom_h264接收处理 start***************************/ static int receiver_custom_h264_check_rtpframe_integrated(rtpframe_t *frame) { if (!list_empty((const struct list_head *)(&frame->part_list))) { rtpframe_part_t *first = list_first_entry(&frame->part_list, rtpframe_part_t, entry); rtpframe_part_t *last = list_last_entry(&frame->part_list, rtpframe_part_t, entry); if (first->psc && last->mark) { rtpframe_part_t *pos; unsigned short seq = first->seq; list_for_each_entry(pos, &frame->part_list, rtpframe_part_t, entry) { if (seq == pos->seq) { ++seq; } else { return 0; } } return 1; } } return 0; } static int receiver_custom_h264_check_rtpframe_partleft(rtpframe_t *frame) { if (!list_empty((const struct list_head *)(&frame->part_list))) { rtpframe_part_t *first = list_first_entry(&frame->part_list, rtpframe_part_t, entry); rtpframe_part_t *last = list_last_entry(&frame->part_list, rtpframe_part_t, entry); if (last->mark && (0 != first->psc)) { return 1; } } return 0; } //根据ts查找缓存rtpframe_t位置 static rtpframe_t *receiver_custom_h264_find_rtpframe(videortp_t *rtp, unsigned ts) { if (rtp->frame_duration == 0) { return &rtp->frame_win[rtp->frame_win_ptr]; } else { unsigned int frame_win_duration = rtp->frame_duration*FRAME_WIN_SIZE; if (RTP_SEQ_GT(ts, rtp->prev_dec_ts, frame_win_duration)) { unsigned int idx = (ts - rtp->prev_dec_ts) / rtp->frame_duration; idx = (rtp->frame_win_ptr + idx + FRAME_WIN_SIZE - 1) % FRAME_WIN_SIZE; return &rtp->frame_win[idx]; } } return NULL; } static int receiver_custom_h264_append_rtpframe_part(videortp_t *rtp, rtpframe_t *frame, unsigned key, unsigned ts, rtpframe_part_t *new_part) { if (frame->used && frame->ts != ts) { rtpframe_clear(rtp, frame); } if (frame->used == 0) { frame->ts = ts; frame->key = key; } else { rtpframe_part_t *pos; if (frame->used >= 1 && !key) { OutputDebugStringA("a"); } list_for_each_entry(pos, &frame->part_list, rtpframe_part_t, entry) { if (pos->seq == new_part->seq) { return 0; } else if (RTP_SEQ_GT(pos->seq, new_part->seq, RTP_MAX_JUMP)) { __list_add(&new_part->entry, pos->entry.prev, &pos->entry); frame->used++; return 1; } else if (RTP_SEQ_GT(new_part->seq, pos->seq, RTP_MAX_JUMP)) { continue; } else { return 0; } } } list_add_tail(&new_part->entry, &frame->part_list); frame->used++; return 1; } static int receiver_custom_h264_decode(videortp_t *vrtp, rtpframe_t *frame) { if (frame->used) { rtpframe_part_t *pos = NULL; int got_picture = 0; uint8_t *input_buf = NULL; int input_len = 0; input_buf = (uint8_t*)_alloca(frame->used * MAX_RECV_BUFFER_SIZE+FF_INPUT_BUFFER_PADDING_SIZE); list_for_each_entry(pos, &frame->part_list, rtpframe_part_t, entry) { memcpy(input_buf+input_len, &pos->buf[pos->offset], pos->actual_len); input_len += pos->actual_len; } memset(input_buf+input_len, 0, FF_INPUT_BUFFER_PADDING_SIZE); if(DEBUG_RX){ rtpDbg(vrtp,"%s:%d: receiver_custom_h264_decode frame_used:%d frame_len:%d now:%I64d.", __FILE__, __LINE__, frame->used, input_len, TimeInMilliseconds()); } //decode return decoder_process(vrtp, input_buf, input_len); } return 0; } rtpframe_t *receiver_custom_h264_assemble(videortp_t *vrtp, rtpframe_part_t *part_data, int len, unsigned short seq, unsigned mark, unsigned ts){ h26x_hdr *hdr = (h26x_hdr*)&part_data->buf[0]; char *raw_data = (char*)(hdr + 1); rtpframe_t *frame = NULL; int actual_len = len - sizeof(h26x_hdr) + 2; int appended; int key = hdr->key; int sno = hdr->id; int end = hdr->end; if(DEBUG_RX){ rtpDbg(vrtp,"%s:%d: receiver_custom_h264_assemble actual_len:%d seq:%d key:%d sno:%d end: %d.", __FILE__, __LINE__, actual_len, seq, !!key, sno, end); } part_data->actual_len = actual_len; part_data->offset = sizeof(h26x_hdr)-2; part_data->mark = mark; part_data->seq = seq; part_data->psc = (sno == 0); frame = receiver_custom_h264_find_rtpframe(vrtp, ts); if (!frame) { receiver_clear_window(vrtp); frame = &vrtp->frame_win[vrtp->frame_win_ptr]; } appended = receiver_custom_h264_append_rtpframe_part(vrtp, frame, key, ts, part_data); if (!appended) { rtpframe_part_destroy(vrtp, part_data); } else if(receiver_custom_h264_check_rtpframe_integrated(frame)) { return frame; } return NULL; } static int receiver_process_custom_h264_packet(videortp_t *vrtp, rtpframe_part_t *part_data, int len, unsigned short seq, unsigned mark, unsigned ts, unsigned pt) { #if 1 VideoJBufferPacketInfo packet_info = {0}; h26x_hdr *hdr = (h26x_hdr*)&part_data->buf[part_data->offset]; int actual_len = part_data->actual_len - sizeof(h26x_hdr) + 2; int key = hdr->key; int sno = hdr->id; int end = hdr->end; int offset = part_data->offset + sizeof(h26x_hdr)-2; packet_info.length = actual_len; packet_info.packet_type = NORMAL_PACKET_IN_FRAME; if (sno == 0){ packet_info.packet_type = FIRST_PACKET_IN_FRAME; } if(end){ packet_info.packet_type = LAST_PACKET_IN_FRAME; } packet_info.seq = seq; packet_info.timestamp = ts; packet_info.is_key = key; packet_info.pt = pt; if(DEBUG_RX){ rtpDbg(vrtp,"%s:%d: receiver_process_custom_h264_packet len:%d seq:%d mark:%d ts:%d pt:%u key:%d sno:%d end:%d offset:%d actual_len:%d.", __FILE__, __LINE__, len, seq, mark, ts, pt, key, sno, end, offset, actual_len); } video_jbuffer_api_receive_packet(vrtp->jbuffer, &part_data->buf[offset], &packet_info); rtpframe_part_destroy(vrtp, part_data); #else rtpframe_t *frame = NULL; if(DEBUG_RX){ rtpDbg(vrtp,"%s:%d: receiver_process_custom_h264_packet len:%d seq:%d mark:%d ts:%d.", __FILE__, __LINE__, len, seq, mark, ts); } frame = receiver_custom_h264_assemble(vrtp, part_data, len, seq, mark, ts); if (frame != NULL) { int decoded; do { decoded = 0; if (frame->key || vrtp->prev_dec_seq+1 == rtpframe_get_first_seq(frame)) { decoded = receiver_custom_h264_decode(vrtp, frame); if (decoded) { if (vrtp->frame_duration==0 && vrtp->prev_dec_seq+1 == rtpframe_get_first_seq(frame)) { vrtp->frame_duration = frame->ts - vrtp->prev_dec_ts; rtpDbg(vrtp,"%s:%d: receiver_process_custom_h264_packet decode frame duration: %d.", __FILE__, __LINE__, vrtp->frame_duration); } vrtp->prev_dec_seq = rtpframe_get_last_seq(frame); vrtp->prev_dec_ts = frame->ts; } else { rtpDbg(vrtp,"%s:%d: receiver_process_custom_h264_packet decode_frame failed!", __FILE__, __LINE__); } rtpframe_clear(vrtp, frame); vrtp->frame_win_ptr = (frame->id+1) % FRAME_WIN_SIZE; frame = &vrtp->frame_win[vrtp->frame_win_ptr]; } else { if (!frame->key) { receiver_notify_peer_fast_update(vrtp); } } } while (decoded && frame->used && receiver_custom_h264_check_rtpframe_integrated(frame)); } #endif return 0; } /******************************custom_h264接收处理 end***************************/ #if 0 #endif static void receiver_process_rtcp_packet(videortp_t *vrtp, unsigned rtcp_flags){ if (rtcp_flags & RTP_SESSION_HAS_FIR){ receiver_force_key_frame(vrtp); } if (rtcp_flags & RTP_SESSION_HAS_RTPFB_TMMBR) { rtcp_statistics rtcp_stats = {0}; rtp_session_get_rtcp_stat(vrtp->rtp_sess, &rtcp_stats); sim_recv_bitrate_feedback(rtcp_stats.tmmbr_max_bitrate); vrtp->recv_cc_feedback_ms = TimeInMilliseconds(); vrtp->recv_cc_feedback = TRUE; vrtp->remote_support_cc = TRUE; } if ((rtcp_flags & RTP_SESSION_HAS_RR) || (rtcp_flags & RTP_SESSION_HAS_SR)) { rtcp_statistics rtcp_stats = {0}; rtp_session_get_rtcp_stat(vrtp->rtp_sess, &rtcp_stats); sim_update_rtt((uint32_t)(rtcp_stats.rtt*1000)); sim_recv_fraction_loss_feedback(rtcp_stats.fraction_lost, rtcp_stats.report_block_last_number_of_packets); } } #ifdef _WIN32 static unsigned int __stdcall recv_proc(void* arg) #else void* recv_proc(void* arg) #endif { videortp_t *vrtp = (videortp_t*)arg; int ifresh_time = vrtp->config.fresh_time; int rtp_timeout_cnt = 0; int64_t pre_500_ms = TimeInMilliseconds(); rtpDbg(vrtp,"enter recv_proc,and ifresh time is %dms.", ifresh_time); #ifdef _WIN32 SetThreadPriority(GetCurrentThread(), THREAD_PRIORITY_ABOVE_NORMAL); #else struct sched_param param; int policy; pthread_getschedparam(pthread_self(), &policy, ¶m); rtpDbg(vrtp,"%s:%d current thread policy is %d, priority is %d.", __FUNCTION__, __LINE__, policy, param.sched_priority); #endif receiver_notify_peer_fast_update_now(vrtp); for (;;) { #ifdef _WIN32 DWORD dwRet = WaitForSingleObject(vrtp->evt, 10); if (dwRet == WAIT_TIMEOUT) #else struct timespec ts; //int ivalue = -1; clock_gettime(CLOCK_REALTIME, &ts); long unsec = ts.tv_nsec + (1000 * 1000 * ifresh_time); ts.tv_sec += (unsec / 1000000000); ts.tv_nsec = (unsec % 1000000000); if (0 != sem_timedwait(&vrtp->sem_evt, &ts) && (ETIMEDOUT == errno)) #endif { int n; { #ifdef _WIN32 unsigned rtcp_flags = 0; int result = rtp_session_recv_rtcp(vrtp->rtp_sess, &rtcp_flags); if (result >= 0) { receiver_process_rtcp_packet(vrtp, rtcp_flags); } #endif } do { unsigned short seq = 0; unsigned mark = 0; unsigned extension = 0; unsigned ts = 0; unsigned pt = 0; rtp_hdr hdr = {0}; rtpframe_part_t *part_data = rtpframe_part_create(vrtp); rtp_session_t *rtp_sess = vrtp->rtp_sess; part_data->actual_len = 0; part_data->mark = 0; part_data->psc = 0; part_data->seq = 0; part_data->offset = 0; part_data->mb_in_slice = -1; n = rtp_session_recv_hook2(rtp_sess, &hdr, part_data->buf, sizeof(part_data->buf), vrtp->config.on_rx_udp, vrtp->config.user_data); seq = ntohs(hdr.seq); mark = hdr.m; extension = hdr.x; ts = ntohl(hdr.ts); pt = hdr.pt; #if 0 // for testing #define PACKET_LOSS_RATIO 30 if (n > 0 && rand()%1000 >= PACKET_LOSS_RATIO) { rtpDbg(vrtp,"%s:%d rtp_session_recv_hook2 pt = %u, mark = %u, ts = %u, seq = %u, size = %d", __FUNCTION__, __LINE__, pt, mark, ts, seq, n); #else if (n > 0) { #endif { uint32_t send_time = 0; if (vrtp->rtp_header_extension && extension){ int extension_length = 0; rtp_header_extension_api_decode_absolute_send_time(vrtp->rtp_header_extension, (uint8_t *)part_data->buf, n, &send_time, &extension_length); part_data->actual_len = n - extension_length; part_data->offset = extension_length; if(DEBUG_RX){ rtpDbg(vrtp,"%s:%d: recv_proc rtp_header_extension send_time:%u length:%d actual_len:%d offset:%d.", __FILE__, __LINE__, send_time, extension_length, part_data->actual_len, part_data->offset); } }else { part_data->actual_len = n; part_data->offset = 0; } //huchen add for video cc #ifdef _WIN32 { sim_segment_t seg = { 0 }; seg.transport_seq = seq; //注意，cc中的ts是以真实时间毫秒为单位，rtp的时间戳是90000为1秒，这里需要转换 seg.timestamp = (send_time == 0 ? (ts / (VIDEO_CLOCK / 1000)) : send_time); seg.data_size = n; sim_recv_video(&seg); } #endif } video_stats_bandwidth_update_recv_rtp_bytes(n); video_stats_receiver_on_incoming_packet(TimeInMilliseconds()); if (REC_COMMON_VIDEO_PT == pt){ receiver_process_custom_h264_packet(vrtp, part_data, n, seq, mark, ts, pt); }else{ receiver_process_standard_h264_packet(vrtp, part_data, n, seq, mark, ts, pt); } rtp_timeout_cnt = 0; } else { rtpframe_part_destroy(vrtp, part_data); } } while (n > 0); { //1s没有解码到视频数据，请求I帧 int64_t now_ms = TimeInMilliseconds(); if (now_ms - vrtp->prev_dec_ms >= 1000){ //rtpDbg(vrtp,"recv_proc now:%I64d > prev_dec_ms:%I64d 1s.", TimeInMilliseconds(), vrtp->prev_dec_ms); receiver_notify_peer_fast_update_now(vrtp); vrtp->prev_dec_ms = now_ms; } if (now_ms - pre_500_ms >= 500){ //发送流控信令,表明接收端支持拥塞控制，让发送端发送加上send time(rtp header extension) if (vrtp->send_cc_feedback == FALSE){ int max_bitrate, target_bitrate, min_bitrate; get_start_config_bitrate(&vrtp->config, &max_bitrate, &target_bitrate, &min_bitrate); rtp_session_send_rtcp_fb_tmmbr(vrtp->rtp_sess, target_bitrate); } pre_500_ms = now_ms; } } } else { break; } } rtpDbg(vrtp, "exit recv_proc."); return 0; } /******************************custom_h264发送处理 start***************************/ static void sender_custom_h264_packetize_and_send(videortp_t *vrtp, int key, unsigned pt, const uint8_t *buf, int n) { const char *ptr = (const char*)buf; int left = n; unsigned delta_ts = vrtp->delta_ts; int id = 0; int mtu = get_mtu(vrtp->config.mtu); mtu = mtu-20-8-12-sizeof(h26x_hdr)+2; // ip hdr: 20, udp hdr: 8, rtp hdr: 12 if(DEBUG_TX){ rtpDbg(vrtp,"%s:%d: sender_custom_h264_packetize_and_send key: %d len: %d delta_ts: %d now: %I64d.", __FILE__, __LINE__, key, n, delta_ts, TimeInMilliseconds()); } video_debug_write_record_file(VIDEO_ENC_OUT, buf, n); while (left > 0) { int rc; char tx_buf[MAX_PLAYLOAD_SIZE]; h26x_hdr *hdr = (h26x_hdr*)&tx_buf[0]; unsigned rtcp_flag; int tn = min(left, mtu); unsigned mark = (tn == left); hdr->key = !!key; hdr->end = mark; hdr->id = id++; memcpy((void *)(&tx_buf[0]+sizeof(h26x_hdr)-2), (const void *)ptr, (size_t)tn); if (DEBUG_TX) { rtpDbg(vrtp,"end = %d, id = %d, tn = %d", hdr->end, hdr->id, tn); } rc = rtp_session_send_hook(vrtp->rtp_sess, &rtcp_flag, pt, mark, delta_ts, tx_buf, tn+sizeof(h26x_hdr)-2, vrtp->config.on_tx_udp, vrtp->config.user_data); if (rc == 0) { receiver_process_rtcp_packet(vrtp, rtcp_flag); delta_ts = 0; left -= tn; ptr += tn; } else { break; } } } static void sender_custom_h264_send_fragment(videortp_t *vrtp, int index, int key_frame, unsigned mark, unsigned pt, uint8_t *fragment, int length) { char tx_buf[MAX_PLAYLOAD_SIZE] = {0}; video_debug_write_record_file(VIDEO_ENC_OUT, fragment, length); video_stats_bandwidth_update_send_rtp_bytes(length); video_stats_sender_on_sent_packet(TimeInMilliseconds()); if (MAX_PLAYLOAD_SIZE > length + sizeof(h26x_hdr)-2 + 32/*rtp_header_extension_length*/){ int rc; unsigned rtcp_flag; h26x_hdr *hdr; int rtp_extension = 0; char rtp_header_extension_buf[32] = {0}; int rtp_header_extension_length = 0; unsigned int delta_ts = vrtp->delta_ts; if(DEBUG_TX){ rtpDbg(vrtp,"%s:%d: sender_custom_h264_send_fragment mark: %d length: %d delta_ts: %d now: %I64d.", __FILE__, __LINE__, mark, length, delta_ts, TimeInMilliseconds()); } //rtp_header_extension if (vrtp->rtp_header_extension && vrtp->remote_support_cc){ uint32_t interval_ms = TimeInMilliseconds() - vrtp->rtp_start_ms; rtp_header_extension_api_encode_absolute_send_time(vrtp->rtp_header_extension, interval_ms, (uint8_t *)rtp_header_extension_buf, &rtp_header_extension_length); memcpy((void *)(&tx_buf[0]), (const void *)rtp_header_extension_buf, (size_t)rtp_header_extension_length); rtp_extension = 1; if(DEBUG_TX){ rtpDbg(vrtp,"%s:%d: sender_custom_h264_send_fragment rtp_header_extension interval_ms:%u length:%d.", __FILE__, __LINE__, interval_ms, rtp_header_extension_length); } } //custom header hdr = (h26x_hdr*)&tx_buf[rtp_header_extension_length]; hdr->key = !!key_frame; hdr->end = mark; hdr->id = index; //playload memcpy((void *)(&tx_buf[rtp_header_extension_length]+sizeof(h26x_hdr)-2), (const void *)fragment, (size_t)length); { rtp_hdr rtp_hdr; rtp_state *rtp_state; rtp_state = rtp_session_get_rtp_state(vrtp->rtp_sess); rtp_state_fill_rtp(rtp_state, &rtp_hdr, pt, mark, delta_ts); rtp_hdr.x = rtp_extension; rc = rtp_session_send_hook2(vrtp->rtp_sess, &rtcp_flag, &rtp_hdr, tx_buf, length+sizeof(h26x_hdr)-2+rtp_header_extension_length, vrtp->config.on_tx_udp, vrtp->config.user_data); if (rc == 0) { receiver_process_rtcp_packet(vrtp, rtcp_flag); } } } } /******************************custom_h264发送处理 end***************************/ /******************************standard_h264发送处理 start***************************/ static int sender_standard_h264_send_packet(videortp_t *vrtp, unsigned mark, unsigned pt, const uint8_t *packet, int length) { char tx_buf[MAX_PLAYLOAD_SIZE] = {0}; if (MAX_PLAYLOAD_SIZE > length + 32/*rtp_header_extension_length*/){ int rc; unsigned rtcp_flag; int rtp_extension = 0; char rtp_header_extension_buf[32] = {0}; int rtp_header_extension_length = 0; unsigned int delta_ts = vrtp->delta_ts; if(DEBUG_TX){ rtpDbg(vrtp,"%s:%d: sender_standard_h264_send_packet mark: %d length: %d delta_ts: %d now: %I64d.", __FILE__, __LINE__, mark, length, delta_ts, TimeInMilliseconds()); } //rtp_header_extension if (vrtp->rtp_header_extension && vrtp->remote_support_cc){ uint32_t interval_ms = TimeInMilliseconds() - vrtp->rtp_start_ms; rtp_header_extension_api_encode_absolute_send_time(vrtp->rtp_header_extension, interval_ms, (uint8_t *)rtp_header_extension_buf, &rtp_header_extension_length); memcpy(&tx_buf[0], rtp_header_extension_buf, rtp_header_extension_length); rtp_extension = 1; if(DEBUG_TX){ rtpDbg(vrtp,"%s:%d: sender_standard_h264_send_packet rtp_header_extension interval_ms:%u length:%d.", __FILE__, __LINE__, interval_ms, rtp_header_extension_length); } } //playload memcpy(&tx_buf[rtp_header_extension_length], packet, length); { rtp_hdr rtp_hdr; rtp_state *rtp_state; rtp_state = rtp_session_get_rtp_state(vrtp->rtp_sess); rtp_state_fill_rtp(rtp_state, &rtp_hdr, pt, mark, delta_ts); rtp_hdr.x = rtp_extension; rc = rtp_session_send_hook2(vrtp->rtp_sess, &rtcp_flag, &rtp_hdr, tx_buf, length + rtp_header_extension_length, vrtp->config.on_tx_udp, vrtp->config.user_data); if (rc == 0) { receiver_process_rtcp_packet(vrtp, rtcp_flag); } } } return 0; } // unsigned short id : 5; // unsigned short end : 2; // unsigned short key : 1; static int sender_standard_h264_send_nal(videortp_t *vrtp, unsigned mark, unsigned pt, uint8_t *nal_payload, int length) { video_debug_write_record_file(VIDEO_ENC_OUT, nal_payload, length); video_stats_bandwidth_update_send_rtp_bytes(length); video_stats_sender_on_sent_packet(TimeInMilliseconds()); if(vrtp->h264_packetizer){ const uint8_t *payload; size_t payload_len; int status; unsigned tmp_mark; //h264_packetize func change nal_payload, so we need copy it to tx_buf first, let nal_payload unchanged. unsigned int processed = 0; uint8_t *tx_buf = (uint8_t*)_alloca(length); memcpy(tx_buf, nal_payload, length); while(processed < length){ status = h264_packetize(vrtp->h264_packetizer, (uint8_t *)tx_buf, length, &processed, &payload, &payload_len); if (status != 0) { return status; } if(DEBUG_TX){ rtpDbg(vrtp,"%s:%d: sender_standard_h264_send_nal mark: %d processed: %u payload_len: %d.", __FILE__, __LINE__, mark, processed, payload_len); } //input mark =1, means last nal, rtp mark = 1, we need last nal 's last packet. if (mark && (processed >= length)){ tmp_mark = 1; } else { tmp_mark = 0; } sender_standard_h264_send_packet(vrtp, tmp_mark, pt, payload, payload_len); } } else { int tn = length; int start_code_length = 0; //comment, remove nal start code if (nal_payload && nal_payload[0] == 0 && nal_payload[1] == 0 && nal_payload[2] == 0 && nal_payload[3] == 1) { start_code_length = 4; if (length - start_code_length > 0) { tn = length - start_code_length; } else { return -1; } } else if (nal_payload && nal_payload[0] == 0 && nal_payload[1] == 0 && nal_payload[2] == 1) { start_code_length = 3; if (length - start_code_length > 0) { tn = length - start_code_length; } else { return -1; } } sender_standard_h264_send_packet(vrtp, mark, pt, nal_payload + start_code_length, tn); } return 0; } static void sender_standard_h264_packetize_x264_nals(videortp_t *vrtp, unsigned pt, x264_nal_t *xnals, int num_nals) { int i; if(DEBUG_TX){ rtpDbg(vrtp,"%s:%d: sender_standard_h264_packetize_x264_nals num_nals: %d.", __FILE__, __LINE__, num_nals); } for (i = 0;i < num_nals;++i){ unsigned mark = 0; x264_nal_t* nal = &xnals[i]; if (xnals[i].i_type == NAL_SPS) { if (DEBUG_TX) { rtpDbg(vrtp,"A SPS is being sent."); } }else if (xnals[i].i_type == NAL_PPS) { if (DEBUG_TX) { rtpDbg(vrtp,"A PPS is being sent."); } }else if (xnals[i].i_type == NAL_SLICE_IDR) { if (DEBUG_TX) { rtpDbg(vrtp,"A IDR is being sent."); } } if (i == num_nals -1){ mark = 1; if (DEBUG_TX) { rtpDbg(vrtp,"A frame end."); } } sender_standard_h264_send_nal(vrtp, mark , pt, nal->p_payload, nal->i_payload); } } void sender_h264_packetize_nal(videortp_t *vrtp, int index, int total, int key_frame, unsigned pt, uint8_t *data, int data_size){ int i; unsigned mark = 0; if(DEBUG_TX){ rtpDbg(vrtp,"%s:%d: sender_h264_packetize_nal index: %d total: %d key_frame: %d data_size: %d.", __FILE__, __LINE__, index, total, key_frame, data_size); } if (index == total -1){ mark = 1; if(DEBUG_TX){ rtpDbg(vrtp,"A frame end."); } } //pt=97 rtp ts = vrtp->delta_ts //pt=101 rtp ts += vrtp->delta_ts if (index == 0){ if (pt == REC_COMMON_VIDEO_PT){ vrtp->delta_ts = VIDEO_CLOCK * vrtp->config.fps_den / vrtp->config.fps_num; } else { vrtp->delta_ts += VIDEO_CLOCK * vrtp->config.fps_den / vrtp->config.fps_num; } }else { if (pt == REC_COMMON_VIDEO_PT){ vrtp->delta_ts = 0; } else { //do nothing } } if (pt == REC_COMMON_VIDEO_PT){ sender_custom_h264_send_fragment(vrtp, index, key_frame, mark, pt, data, data_size); } else { sender_standard_h264_send_nal(vrtp, mark, pt, data, data_size); } } /******************************standard_h264发送处理 end***************************/ #if 0 #endif /******************************congestion control处理 start***************************/ int congestion_control_on_log(void *userdata, int level, const char* file, int line, const char* fmt, va_list vl){ videortp_t *vrtp = (videortp_t *)userdata; int n; n = vsnprintf(NULL, 0, fmt, vl); if (DEBUG_CC && n > 0 && level >= 1 && vrtp) { char *buf = (char*)_alloca((size_t)(n+3)); vsprintf(buf, fmt, vl); rtpDbg(vrtp,"%s.", buf); } } /* 发送端回调。运行在sim_heartbeat方法调用线程。sim_send_video->cc缓存,然后sim_heartbeat->pacing判断可以发送->回调 */ void congestion_control_sender_on_send_fragment(sim_segment_t* seg, void *userdata){ videortp_t *vrtp = (videortp_t *)userdata; if(DEBUG_TX){ rtpDbg(vrtp,"%s:%d: congestion_control_sender_on_send_fragment.", __FILE__, __LINE__); } sender_h264_packetize_nal(vrtp, seg->index, seg->total, seg->key_frame, seg->payload_type, seg->data, seg->data_size); } /* 发送端回调。运行在sim_heartbeat方法调用线程。 sim_heartbeat->cc判断带宽估计是否变化->回调 */ /* 或者sim_recv_bitrate_feedback调用线程 . sim_recv_bitrate_feedback->cc判断带宽估计是否变化->回调 */ /*bw 估计码率，单位：kbps*/ void congestion_control_sender_on_change_bitrate(uint32_t bw, int lost, void *userdata){ videortp_t *vrtp = (videortp_t *)userdata; if(DEBUG_CC){ rtpDbg(vrtp,"%s:%d: congestion_control_sender_on_change_bitrate bw: %d lost: %d.", __FILE__, __LINE__, bw, lost); } //回调时编码器可能未创建 if (vrtp->encoder) { video_encoder_api_on_bitrate_updated(vrtp->encoder, bw*1000, lost, 0); } } /* 接收端回调。运行在sim_heartbeat方法调用线程，sim_heartbeat->cc判断需要发送remb->回调 */ void congestion_control_receiver_on_send_bitrate_feedback(uint32_t estimate_bitrate, void *userdata){ videortp_t *vrtp = (videortp_t *)userdata; if(DEBUG_CC){ rtpDbg(vrtp,"%s:%d: congestion_control_receiver_on_send_bitrate_feedback estimate_bitrate: %d.", __FILE__, __LINE__, estimate_bitrate); } video_stats_receiver_on_report_cc_bitrate(estimate_bitrate); rtp_session_send_rtcp_fb_tmmbr(vrtp->rtp_sess, estimate_bitrate); vrtp->send_cc_feedback = TRUE; } void congestion_control_create(videortp_t *vrtp, const videortp_config_t *config){ int max_bitrate, target_bitrate, min_bitrate; get_start_config_bitrate(config, &max_bitrate, &target_bitrate, &min_bitrate); { sim_callback_t callback = {0}; callback.userdata = vrtp; callback.log_cb = &congestion_control_on_log; callback.send_fragment_cb = &congestion_control_sender_on_send_fragment; callback.change_bitrate_cb = &congestion_control_sender_on_change_bitrate; callback.send_bitrate_feedback_cb = &congestion_control_receiver_on_send_bitrate_feedback; sim_init(gcc_transport, 0, min_bitrate, target_bitrate, max_bitrate, &callback); } rtpDbg(vrtp,"congestion_control_create min_bitrate: %d target_bitrate: %d max_bitrate: %d.", min_bitrate, target_bitrate, max_bitrate); } void congestion_control_destroy(videortp_t *vrtp){ sim_destroy(); rtpDbg(vrtp,"congestion_control_destroy."); } /******************************congestion control处理 end***************************/ #if 0 #endif /******************************x264 encoder处理 start***************************/ #ifndef IDR_INTERVAL #define IDR_INTERVAL 8 #endif #ifndef NAL_TIMESTAP_ENC #define NAL_TIMESTAP_ENC 3000 #endif #ifndef SEND_WIDTH #define SEND_WIDTH 320 #endif #ifndef SEND_HEIGHT #define SEND_HEIGHT 320 #endif #ifndef SEND_FPS #define SEND_FPS 15 #endif #ifndef SEND_BITRATE #define SEND_BITRATE 180*1000 #endif #ifndef RC_MARGIN #define RC_MARGIN 10000 #endif #define DEFAULT_MAX_PAYLOAD_SIZE 1440 static int max_payload_size = DEFAULT_MAX_PAYLOAD_SIZE; int x264_encoder_get_payload_max_size(){ return max_payload_size; } static void x264_encoder_set_linephone_param(x264_param_t* params) { float bitrate; int csp = X264_CSP_I420; int iframerate = REC_COMMON_VIDEO_FPS_NUM; params->i_threads=1; params->i_sync_lookahead=0; params->i_width = SEND_WIDTH; params->i_height = SEND_HEIGHT; params->i_fps_num=(int)iframerate; params->i_fps_den=1; params->i_slice_max_size=x264_encoder_get_payload_max_size()-100; /*-100 security margin*/ params->i_level_idc=13; bitrate=(float)SEND_BITRATE*0.92; if (bitrate>RC_MARGIN) bitrate-=RC_MARGIN; #ifndef ANDROID params->rc.i_rc_method = X264_RC_ABR; params->rc.i_bitrate=(int)(bitrate/1000); params->rc.f_rate_tolerance=0.1; params->rc.i_vbv_max_bitrate=(int) ((bitrate+RC_MARGIN/2)/1000); params->rc.i_vbv_buffer_size=params->rc.i_vbv_max_bitrate; params->rc.f_vbv_buffer_init=0.5; #else params.rc.i_rc_method = X264_RC_CQP; params.rc.i_bitrate=(int)(bitrate/1000); #endif params->rc.i_lookahead=0; params->b_repeat_headers=1; params->b_annexb=0; //these parameters must be set so that our stream is baseline params->analyse.b_transform_8x8 = 0; params->b_cabac = 0; params->i_cqm_preset = X264_CQM_FLAT; params->i_bframe = 0; params->analyse.i_weighted_pred = X264_WEIGHTP_NONE; } static void x264_encoder_set_param_t(x264_param_t* params) { float bitrate; int csp = X264_CSP_I420; int iframerate = REC_COMMON_VIDEO_FPS_NUM; x264_param_default(params); x264_param_default_preset(params, "slow" , "zerolatency" ); params->i_threads=1; params->i_sync_lookahead=0; params->i_width = SEND_WIDTH; params->i_height = SEND_HEIGHT; params->i_fps_num = iframerate; params->i_fps_den = 1; params->i_slice_max_size=x264_encoder_get_payload_max_size()-100; /*-100 security margin*/ bitrate=(float)SEND_BITRATE*0.92; if (bitrate>RC_MARGIN){ bitrate-=RC_MARGIN; } // rc params->rc.i_rc_method = X264_RC_ABR; params->rc.i_bitrate = (int)bitrate/1000; //must i_vbv_max_bitrate <= i_bitrate reconfig available params->rc.i_vbv_max_bitrate = (int)((bitrate)/1000) ; params->rc.i_vbv_buffer_size = (int)((bitrate*2)/1000) ; params->rc.f_vbv_buffer_init=0.5; params->rc.b_mb_tree=0;// params->rc.f_rf_constant = 25; params->rc.f_rf_constant_max = 45; params->rc.f_rate_tolerance=0.1; params->rc.b_stat_write = 1; params->rc.i_lookahead=0; params->i_keyint_max = iframerate * 2; params->i_keyint_min = iframerate * 2; params->b_repeat_headers = 1; // params->b_annexb= 1; params->b_cabac = 1; params->i_cqm_preset = X264_CQM_FLAT; params->i_bframe = 0; params->analyse.i_weighted_pred = X264_WEIGHTP_NONE; params->i_csp=csp; params->i_level_idc = 30; x264_param_apply_profile(params, x264_profile_names[0]); params->i_log_level = X264_LOG_WARNING; } void x264_encoder_create(videortp_t *vrtp, const videortp_config_t *config){ x264_param_t params = {0}; x264_param_t params_print = {0}; x264_encoder_set_param_t(¶ms); vrtp->enc_params = x264_encoder_open(¶ms); x264_encoder_parameters(vrtp->enc_params, ¶ms_print); rtpDbg(vrtp,"x264_encoder_create bit_rate: %d kb/s, i_rc_method: %d ", params_print.rc.i_bitrate, params_print.rc.i_rc_method); rtpDbg(vrtp,"x264_encoder_create i_vbv_max_bitrate: %d kb/s, i_vbv_buffer_size: %d ", params_print.rc.i_vbv_max_bitrate, params_print.rc.i_vbv_buffer_size); rtpDbg(vrtp,"x264_encoder_create keyint_max: %d, i_keyint_min: %d ", params_print.i_keyint_max, params_print.i_keyint_min); } void x264_encoder_destroy(videortp_t *vrtp){ if (vrtp->enc_params){ x264_encoder_close(vrtp->enc_params); } rtpDbg(vrtp,"x264_encoder_destroy."); } /******************************x264 encoder处理 end***************************/ #if 0 #endif /******************************ffmpeg encoder处理 start***************************/ void ffmpeg_encoder_on_encoder_encoded_image(const EncodedImage *encoded_image, void *userdata){ int i; videortp_t *vrtp = (videortp_t *)userdata; if (encoded_image != NULL && vrtp != NULL) { if(DEBUG_TX){ rtpDbg(vrtp,"%s:%d: ffmpeg_encoder_on_encoder_encoded_image key: %d len: %d qp: %d, PT:%u.", __FILE__, __LINE__, encoded_image->key_frame, encoded_image->size_, encoded_image->qp_, encoded_image->pt_); rtpDbg(vrtp,"%s:%d: ffmpeg_encoder_on_encoder_encoded_image encode_start_ms: %I64d encode_finish_ms: %I64d fragmentationSize: %d.", __FILE__, __LINE__, encoded_image->encode_start_ms, encoded_image->encode_finish_ms, encoded_image->fragmentation_header_.fragmentationVectorSize); } if(DEBUG_TX){ for (i = 0;i < encoded_image->fragmentation_header_.fragmentationVectorSize;++i){ rtpDbg(vrtp,"%s:%d: ffmpeg_encoder_on_encoder_encoded_image index: %d fragmentationLength: %d.", __FILE__, __LINE__, i, encoded_image->fragmentation_header_.fragmentationLength[i]); } } video_stats_sender_on_encoded_frame(encoded_image->encode_finish_ms, encoded_image->encode_finish_ms - encoded_image->encode_start_ms, encoded_image->key_frame); //采用cc控制发送 if(vrtp->remote_support_cc){ sim_send_video((EncodedImage *)encoded_image); } //不采用cc直接发送 else { int use_fragment_send = 1; //采用新分片发送 if (use_fragment_send){ for (i = 0;i < encoded_image->fragmentation_header_.fragmentationVectorSize;++i){ sender_h264_packetize_nal(vrtp, i, encoded_image->fragmentation_header_.fragmentationVectorSize, encoded_image->key_frame, encoded_image->pt_, &(encoded_image->encoded_data_[encoded_image->fragmentation_header_.fragmentationOffset[i]]), encoded_image->fragmentation_header_.fragmentationLength[i]); } } //采用旧分片发送 else { if (encoded_image->pt_ == REC_COMMON_VIDEO_PT){ sender_custom_h264_packetize_and_send(vrtp, encoded_image->key_frame, encoded_image->pt_, encoded_image->encoded_data_, encoded_image->size_); } else { for (i = 0;i < encoded_image->fragmentation_header_.fragmentationVectorSize;++i){ sender_h264_packetize_nal(vrtp, i, encoded_image->fragmentation_header_.fragmentationVectorSize, encoded_image->key_frame, encoded_image->pt_, &(encoded_image->encoded_data_[encoded_image->fragmentation_header_.fragmentationOffset[i]]), encoded_image->fragmentation_header_.fragmentationLength[i]); } } } } } } void ffmpeg_encoder_create(videortp_t *vrtp, const videortp_config_t *config){ int max_bitrate, target_bitrate, min_bitrate; VideoEncoderObserver video_encoder_observer = {0}; VideoEncoderConfig encoder_config = {0}; VideoCodingLogCallBack log_func = {0}; get_start_config_bitrate(config, &max_bitrate, &target_bitrate, &min_bitrate); video_encoder_observer.OnEncodedImage = &ffmpeg_encoder_on_encoder_encoded_image; video_encoder_observer.userdata = vrtp; log_func.log_fn = &video_coding_on_log; log_func.userdata = vrtp; vrtp->encoder = video_encoder_api_new(&video_encoder_observer, &log_func); encoder_config.color_space = VIDEO_FORMAT_I420; encoder_config.encode_id = CODEC_ID_H264; encoder_config.width = config->tx_width; encoder_config.height = config->tx_height; encoder_config.max_bitrate = max_bitrate/1000; //kbps encoder_config.target_bitrate = target_bitrate/1000; //kbps encoder_config.min_bitrate = min_bitrate/1000; //kbps encoder_config.max_framerate = config->fps_num; encoder_config.target_framerate = config->fps_num; encoder_config.min_framerate = config->fps_num > 3 ? 3 : config->fps_num; encoder_config.key_frame_interval = config->fps_num * 2; encoder_config.target_qp = 24; encoder_config.max_qp = 37; encoder_config.min_qp = 10; video_encoder_api_configure_encoder(vrtp->encoder, &encoder_config, get_mtu(config->mtu) - 5/*padding*/ - ((sizeof(h26x_hdr)-2 + 32/*rtp_header_extension_length*/))); rtpDbg(vrtp,"ffmpeg_encoder_create color_space: %d encode_id: %d width: %d height: %d.", encoder_config.color_space, encoder_config.encode_id, encoder_config.width, encoder_config.height); rtpDbg(vrtp,"ffmpeg_encoder_create max_bitrate: %d min_bitrate: %d target_bitrate: %d.", max_bitrate, min_bitrate, target_bitrate); rtpDbg(vrtp,"ffmpeg_encoder_create min_framerate: %d key_frame_interval: %d target_qp: %d.", encoder_config.min_framerate, encoder_config.key_frame_interval, encoder_config.target_qp); } void ffmpeg_encoder_destory(videortp_t *vrtp){ if (vrtp->encoder != NULL) { video_encoder_api_stop(vrtp->encoder); video_encoder_api_destroy(vrtp->encoder); } rtpDbg(vrtp,"ffmpeg_encoder_destory."); } /******************************ffmpeg encoder处理 end***************************/ #if 0 #endif /******************************jitter buffer处理 start***************************/ int jitter_buffer_on_log(void *userdata, int level, const char* file, int line, const char* fmt, va_list vl){ videortp_t *vrtp = (videortp_t *)userdata; int n; n = vsnprintf(NULL, 0, fmt, vl); if (DEBUG_RX && n > 0 && vrtp && level >= VIDEO_JBUFFER_DEBUG_TRACE) { char *buf = (char*)_alloca((size_t)(n+3)); vsprintf(buf, fmt, vl); rtpDbg(vrtp,"%s.", buf); } } void jitter_buffer_create(videortp_t *vrtp, const videortp_config_t *config){ VideoJBufferObserver video_jbuffer_observer; VideoJBufferLogCallBack log_func = {0}; video_jbuffer_observer.OnReceiveEncodedImage = &receiver_h264_on_receive_encoded_image; video_jbuffer_observer.RequestKeyFrame = &receiver_standard_h264_on_jbuffer_request_key_frame; video_jbuffer_observer.userdata = vrtp; log_func.log_fn = &jitter_buffer_on_log; log_func.userdata = vrtp; vrtp->jbuffer = video_jbuffer_api_new(&video_jbuffer_observer, &log_func); rtpDbg(vrtp,"jitter_buffer_create."); } void jitter_buffer_destory(videortp_t *vrtp){ if (vrtp->jbuffer != NULL) { video_jbuffer_api_destroy(vrtp->jbuffer); } rtpDbg(vrtp,"jitter_buffer_destory."); } /******************************jitter buffer处理 end***************************/ static rtp_session_t *videortp_internal_create_rtp_session(const videortp_config_t *config) { rtp_session_t *rtp_sess; struct in_addr addr; int rc; addr.s_addr = config->local_ip; rc = rtp_session_create(inet_ntoa(addr), config->local_rtp_port, 2, &rtp_sess); if (rc != 0){ return 0; } addr.s_addr = config->remote_ip; rtp_session_reset(rtp_sess, config->dir, inet_ntoa(addr), config->remote_rtp_port, config->remote_rtp_port+1); { // set to a larger recv buffer, so that it will not overflow int rtp_fd = -1/*INVALID_SOCKET*/; int buf_size = 1<<20; rtp_session_get_raw_fd(rtp_sess, &rtp_fd, NULL); if (rtp_fd != -1/*INVALID_SOCKET*/) { setsockopt(rtp_fd, SOL_SOCKET, SO_RCVBUF, (const char*)&buf_size, sizeof(buf_size)); setsockopt(rtp_fd, SOL_SOCKET, SO_SNDBUF, (const char*)&buf_size, sizeof(buf_size)); } } return rtp_sess; } int video_stats_on_log(void *userdata, int level, const char* file, int line, const char* fmt, va_list vl){ videortp_t *vrtp = (videortp_t *)userdata; int n; n = vsnprintf(NULL, 0, fmt, vl); if (n > 0 && vrtp) { char *buf = (char*)_alloca((size_t)(n+3)); vsprintf(buf, fmt, vl); rtpDbg(vrtp,"%s.", buf); } } /******************************对外api start***************************/ int videortp_create(const videortp_config_t *config, videortp_t **p_rtp) { int i, rc; videortp_t *vrtp = ZALLOC_T(videortp_t); vrtp->prev_dec_ms = TimeInMilliseconds(); vrtp->recv_cc_feedback_ms = TimeInMilliseconds(); vrtp->send_cc_feedback = 0; vrtp->recv_cc_feedback = 0; vrtp->remote_support_cc = 0; memcpy((void *)(&vrtp->config), (const void *)config, sizeof(videortp_config_t)); rtpDbg(vrtp,"videortp_create local_pt: %d local_ip: %d local_rtp_port: %d.", config->local_pt, config->local_ip, config->local_rtp_port); rtpDbg(vrtp,"videortp_create remote_pt: %d remote_ip: %d remote_rtp_port: %d.", config->remote_pt, config->remote_ip, config->remote_rtp_port); rtpDbg(vrtp,"videortp_create tx_width: %d tx_height: %d.", config->tx_width, config->tx_height); rtpDbg(vrtp,"videortp_create rx_width: %d rx_height: %d.", config->rx_width, config->rx_height); rtpDbg(vrtp,"videortp_create capture_width: %d capture_height: %d.", config->capture_width, config->capture_height); rtpDbg(vrtp,"videortp_create fps_num: %d bit_rate: %d dir: %d.", config->fps_num, config->bit_rate, config->dir); INIT_LIST_HEAD(&vrtp->free_part_list); for (i = 0; i < FRAME_WIN_SIZE; ++i) { rtpframe_t *frame = &vrtp->frame_win[i]; INIT_LIST_HEAD(&frame->part_list); frame->id = i; } vrtp->delta_ts = VIDEO_CLOCK * config->fps_den / config->fps_num; vrtp->rtp_sess = videortp_internal_create_rtp_session(config); if (!vrtp->rtp_sess) { char strMessage[MAX_PATH] = { 0 }; snprintf(strMessage, MAX_PATH, "video rtp create failed and local port is %d.", config->local_rtp_port); rtpLogevent(vrtp, 0, strMessage); goto on_error; } else { char strInfo[MAX_PATH] = { 0 }; snprintf(strInfo, MAX_PATH, "video rtp create success and local port is %d.", config->local_rtp_port); rtpLogevent(vrtp, 0, strInfo); } { VideoStatsLogCallBack log_func; log_func.log_fn = &video_stats_on_log; log_func.userdata = vrtp; video_stats_init(&log_func); } jitter_buffer_create(vrtp, config); vrtp->rtp_header_extension = rtp_header_extension_api_new(); if (config->dir & DIR_RX) { decoder_create(vrtp, config); video_debug_open_play_file(); } if (config->dir & DIR_TX) { ffmpeg_encoder_create(vrtp, config); video_debug_open_record_file(); vrtp->framenum = 0; } #ifdef _WIN32 congestion_control_create(vrtp, config); #endif { h264_packetizer_cfg cfg; cfg.mtu = get_mtu(config->mtu); cfg.mode = H264_PACKETIZER_MODE_NON_INTERLEAVED; cfg.unpack_nal_start = 3; h264_packetizer_create(&cfg, &(vrtp->h264_packetizer)); } *p_rtp = vrtp; return 0; on_error: videortp_destroy(vrtp); return -1; } void videortp_destroy(videortp_t *vrtp) { videortp_config_t *config; unsigned short ilocal_port = 0; if (!vrtp){ return; } config = &vrtp->config; rtpDbg(vrtp,"videortp_destroy."); receiver_clear_window(vrtp); while (!list_empty(&vrtp->free_part_list)) { rtpframe_part_t *part = list_first_entry(&vrtp->free_part_list, rtpframe_part_t, entry); list_del(&part->entry); free(part); } #ifdef _WIN32 congestion_control_destroy(vrtp); #endif video_stats_uninit(); jitter_buffer_destory(vrtp); rtp_header_extension_api_destroy(vrtp->rtp_header_extension); h264_packetizer_destroy(vrtp->h264_packetizer); rtp_session_get_local_rtp_port(vrtp->rtp_sess, &ilocal_port); rtp_session_destroy(vrtp->rtp_sess); { char strInfo[MAX_PATH] = { 0 }; snprintf(strInfo, MAX_PATH, "video rtp destroy and local port is %u.", ilocal_port); rtpLogevent(vrtp, 1, strInfo); } vrtp->rtp_sess = NULL; if (config->dir & DIR_RX) { decoder_destroy(vrtp); video_debug_close_play_file(); } if (config->dir & DIR_TX) { ffmpeg_encoder_destory(vrtp); video_debug_close_record_file(); } free(vrtp); } int videortp_start(videortp_t *vrtp) { int rc; ReceiverConfigStats receiver_config_stats; SenderConfigStats sender_config_stats; int64_t now = TimeInMilliseconds(); vrtp->rtp_start_ms = now; video_jbuffer_api_start(vrtp->jbuffer); if (vrtp->config.dir & DIR_RX) { #ifdef _WIN32 vrtp->evt = CreateEventA(NULL, FALSE, FALSE, NULL); vrtp->recv_thread = (HANDLE)_beginthreadex(NULL, 0, &recv_proc, vrtp, 0, NULL); #else sem_init(&vrtp->sem_evt, 0, 0); pthread_attr_t attr; pthread_attr_init(&attr); struct sched_param param; param.sched_priority = sched_get_priority_max(SCHED_RR); pthread_attr_setschedpolicy(&attr, SCHED_RR); pthread_attr_setschedparam(&attr, ¶m); pthread_attr_setinheritsched(&attr, PTHREAD_EXPLICIT_SCHED); int err = pthread_create(&vrtp->recv_threadid, NULL, recv_proc, vrtp); if (0 == err) { rtpDbg(vrtp,"create vrtp recv thread success, and thread id is %u.", vrtp->recv_threadid); } else { rtpDbg(vrtp,"create vrtp recv thread failed for %s.", strerror(errno)); } #endif } rtpDbg(vrtp,"videortp_start."); receiver_config_stats.remote_pt = vrtp->config.remote_pt; receiver_config_stats.local_ip = vrtp->config.local_ip; receiver_config_stats.local_rtp_port = vrtp->config.local_rtp_port; receiver_config_stats.rx_width = vrtp->config.rx_width; receiver_config_stats.rx_height = vrtp->config.rx_height; receiver_config_stats.bit_rate = vrtp->config.bit_rate; video_stats_receiver_on_started(now, &receiver_config_stats); sender_config_stats.local_pt = vrtp->config.local_pt; sender_config_stats.remote_ip = vrtp->config.remote_ip; sender_config_stats.remote_rtp_port = vrtp->config.remote_rtp_port; sender_config_stats.mtu = vrtp->config.mtu; sender_config_stats.tx_height = vrtp->config.tx_height; sender_config_stats.tx_width = vrtp->config.tx_width; sender_config_stats.capture_width = vrtp->config.capture_width; sender_config_stats.capture_height = vrtp->config.capture_height; sender_config_stats.fps_den = vrtp->config.fps_den; sender_config_stats.fps_num = vrtp->config.fps_num; sender_config_stats.fps_num = vrtp->config.fps_num; sender_config_stats.bit_rate = vrtp->config.bit_rate; video_stats_sender_on_started(now, &sender_config_stats); return 0; } void videortp_stop(videortp_t* vrtp) { int64_t now = TimeInMilliseconds(); rtpDbg(vrtp,"%s:%d start call videortp_stop.", __FUNCTION__, __LINE__); #ifdef _WIN32 if (vrtp->evt) { SetEvent(vrtp->evt); if (vrtp->recv_thread) { WaitForSingleObject(vrtp->recv_thread, INFINITE); CloseHandle(vrtp->recv_thread); vrtp->recv_thread = NULL; } CloseHandle(vrtp->evt); vrtp->evt = NULL; } #else if (&vrtp->sem_evt) { sem_post(&vrtp->sem_evt); if (0 != vrtp->recv_threadid) { if (0 == pthread_join(vrtp->recv_threadid, NULL)){ rtpDbg(vrtp, "%s:%d vrtp receive thread %u thread join success.", __FUNCTION__, __LINE__, vrtp->recv_threadid); vrtp->recv_threadid = 0; } else { rtpDbg(vrtp, "%s:%d vrtp receive thread thread join failed for %s.", __FUNCTION__, __LINE__, strerror(errno)); } } sem_destroy(&vrtp->sem_evt); } #endif rtpDbg(vrtp,"%s:%d stop rtp recv thread.", __FUNCTION__, __LINE__); video_stats_receiver_on_stopped(now); video_stats_sender_on_stopped(now); video_jbuffer_api_stop(vrtp->jbuffer); rtpDbg(vrtp,"%s:%d videortp_stop func end.", __FUNCTION__, __LINE__); } int videortp_send_frame(videortp_t *vrtp, const video_frame *frame) { if (vrtp && (vrtp->config.dir & DIR_TX)) { int64_t now; if(DEBUG_TX){ rtpDbg(vrtp,"%s:%d: videortp_send_frame width: %d height: %d format: %d.", __FILE__, __LINE__, frame->width, frame->height, frame->format); } if (VIDEO_FORMAT_RGB24 != frame->format) { unsigned char *video_debug_buf; int length = frame->width*frame->height*3/2; video_debug_buf = (unsigned char *)malloc(length); yuv_buffers_combine_to_one(frame, video_debug_buf); video_debug_write_record_file(VIDEO_CAPTURE_OUT, video_debug_buf, length); free(video_debug_buf); } if (vrtp->force_key_frames) { video_encoder_api_send_key_frame(vrtp->encoder); vrtp->force_key_frames = 0; if(DEBUG_TX){ rtpDbg(vrtp,"%s:%d: videortp_send_frame force_key_frames.", __FILE__, __LINE__); } } //在编码器中固定i帧间隔为2s，不再需要自己主动设置了 #if 0 else { int start_key_interval = vrtp->config.fps_num / vrtp->config.fps_den / 2; if (start_key_interval == 0) start_key_interval = 1; if (vrtp->framenum/start_key_interval <= FIRST_KEY_FRAME_SIZE) { if (vrtp->framenum % start_key_interval == 0) { video_encoder_api_send_key_frame(vrtp->encoder); if(DEBUG_TX){ rtpDbg(vrtp,"%s:%d: videortp_send_frame framenum: %d force_key_frames.", __FILE__, __LINE__, vrtp->framenum); } } } } #endif video_encoder_api_on_frame(vrtp->encoder, vrtp->config.local_pt, frame); vrtp->framenum++; //从未收到cc_feedback估计带宽，远端不支持流量控制，将码率设为配置值 #ifdef _WIN32 now = TimeInMilliseconds(); if ((vrtp->recv_cc_feedback == FALSE) && (now - vrtp->recv_cc_feedback_ms >= 10 * 1000)) { rtpDbg(vrtp, "videortp_send_frame, never receive cc feedback, remote don't support ."); sim_recv_bitrate_feedback(vrtp->config.bit_rate/*bps*/); vrtp->recv_cc_feedback = TRUE; } #endif } else { return -1; } return 0; } int videortp_send_yuvframe(videortp_t *vrtp, const video_frame *vframe) { videortp_send_frame(vrtp, vframe); //static int itest = 0; //if (itest == 0) { // video_frame_save_bmpfile("videortp_send_yuvframe.bmp", vframe); // itest++; //} return 0; } int videortp_reconfig_local_pt(videortp_t *vrtp, unsigned int local_pt){ rtpDbg(vrtp,"videortp_reconfig_local_pt:%u .", local_pt); vrtp->config.local_pt = local_pt; } bool videortp_thread_valid(videortp_t* vrtp) { bool bret = true; #ifdef _WIN32 bret = vrtp->recv_thread != NULL ? true : false; #else bret = vrtp->recv_threadid != 0 ? true : false; #endif return bret; } int videortp_force_key_frame(videortp_t *vrtp) { int iret = -1; if (videortp_thread_valid(vrtp)) { receiver_force_key_frame(vrtp); iret = 0; } return iret; } /******************************对外api end***************************/