#include "player.h" #include "packet.h" #include "frame.h" #ifdef _WIN32 #include "Windows.h" #endif static void sdl_audio_callback(void *opaque, uint8_t*stream, int len); // 从packet_queue中取一个packet,解码生成frame static int audio_decode_frame(AVCodecContext *p_codec_ctx, packet_queue_t *p_pkt_queue, AVFrame *frame, CMediaHostApi* hostapi) { int ret = -1; if (NULL == p_codec_ctx){ return ret; } while (0 == p_pkt_queue->abort_flag) { AVPacket pkt = {0}; while (0 == p_pkt_queue->abort_flag) { // 3.2 一个音频packet含一至多个音频frame,每次avcodec_receive_frame()返回一个frame,此函数返回。 // 下次进来此函数,继续获取一个frame,直到avcodec_receive_frame()返回AVERROR(EAGAIN), // 表示解码器需要填入新的音频packet ret = avcodec_receive_frame(p_codec_ctx, frame); if (ret >= 0) { // 时基转换,从d->avctx->pkt_timebase时基转换到1/frame->sample_rate时基 AVRational tb = { 1, frame->sample_rate }; if (frame->pts != AV_NOPTS_VALUE) { frame->pts = av_rescale_q(frame->pts, p_codec_ctx->pkt_timebase, tb); } else { //hostapi->Debug(MEDIA_LOG_DEBUG, "frame->pts no."); } return 1; } else if (ret == AVERROR_EOF) { avcodec_flush_buffers(p_codec_ctx); return -1; } else if (ret == AVERROR(EAGAIN)) { break; } else { hostapi->Debug(MEDIA_LOG_DEBUG, "audio avcodec_receive_frame(): other errors."); continue; } } // 1. 取出一个packet。使用pkt对应的serial赋值给d->pkt_serial if (packet_queue_get(p_pkt_queue, &pkt, true, hostapi) < 0) { return -1; } // packet_queue中第一个总是flush_pkt。每次seek操作会插入flush_pkt,更新serial,开启新的播放序列 if (NULL == pkt.data || 0 == pkt.size) { // 复位解码器内部状态/刷新内部缓冲区。当seek操作或切换流时应调用此函数。 avcodec_flush_buffers(p_codec_ctx); return -2; } else { // 2. 将packet发送给解码器 // 发送packet的顺序是按dts递增的顺序,如IPBBPBB // pkt.pos变量可以标识当前packet在视频文件中的地址偏移 int iresult = avcodec_send_packet(p_codec_ctx, &pkt); if (AVERROR(EAGAIN) == iresult) { hostapi->Debug(MEDIA_LOG_DEBUG, "receive_frame and send_packet both returned EAGAIN, which is an API violation."); } if (0 == iresult) { av_packet_unref(&pkt); } } } return ret; } // 音频解码线程:从音频packet_queue中取数据,解码后放入音频frame_queue static int audio_decode_thread(void *arg) { player_stat_t *is = (player_stat_t *)arg; AVFrame *p_frame = av_frame_alloc(); frame_t *af; int got_frame = 0; AVRational tb; int ret = 0; if (p_frame == NULL){ return AVERROR(ENOMEM); } while (false == is->buser_stop) { got_frame = audio_decode_frame(is->m_pacodec_ctx[is->m_iaudio_dec_index], &is->audio_pkt_queue, p_frame, is->rvc_hostapi); if (got_frame < 0){ if(-2 == got_frame){ if (is->m_icurrent_index > is->m_iaudio_dec_index) { is->m_iaudio_dec_index++; } continue; } else { goto the_end; } } if (got_frame) { tb.num = 1; tb.den = p_frame->sample_rate; //从frame队列找到一个可写的空间,若未停止则一直等待,已停止时返回NULL if (!(af = frame_queue_peek_writable(&is->audio_frm_queue))) { goto the_end; } af->pts = (p_frame->pts == AV_NOPTS_VALUE) ? NAN : p_frame->pts * av_q2d(tb); af->pos = p_frame->pkt_pos; //-af->serial = is->auddec.pkt_serial; // 当前帧包含的(单个声道)采样数/采样率就是当前帧的播放时长 AVRational tbdata = { p_frame->nb_samples, p_frame->sample_rate }; //af->duration = av_q2d((AVRational) { p_frame->nb_samples, p_frame->sample_rate }); af->duration = av_q2d(tbdata); // 将frame数据拷入af->frame,af->frame指向音频frame队列尾部 av_frame_move_ref(af->frame, p_frame); // 更新音频frame队列大小及写指针 frame_queue_push(&is->audio_frm_queue); } } the_end: av_frame_free(&p_frame); is->rvc_hostapi->Debug(MEDIA_LOG_DEBUG, "audio decode thread exit, thread id is %u, and user stop flag is %s.", SDL_ThreadID(), is->buser_stop ? "true":"false"); is->m_baudio_decode_finished = true; return ret; } int open_audio_stream(player_stat_t *is) { AVCodecContext *p_codec_ctx = NULL; AVCodecParameters *p_codec_par = NULL; AVCodec* p_codec = NULL; int ret = -1; // 1. 为音频流构建解码器AVCodecContext for (size_t index = 0; index < is->m_uFilesCount; index++){ // 1.1 获取解码器参数AVCodecParameters p_codec_par = is->m_paudio_stream[index]->codecpar; // 1.2 获取解码器 p_codec = avcodec_find_decoder(p_codec_par->codec_id); if (NULL == p_codec) { is->rvc_hostapi->Debug(MEDIA_LOG_ERROR, "Cann't find codec!"); return ret; } // 1.3 构建解码器AVCodecContext // 1.3.1 p_codec_ctx初始化:分配结构体,使用p_codec初始化相应成员为默认值 p_codec_ctx = avcodec_alloc_context3(p_codec); if (p_codec_ctx == NULL) { is->rvc_hostapi->Debug(MEDIA_LOG_ERROR, "avcodec_alloc_context3() failed."); return ret; } // 1.3.2 p_codec_ctx初始化:p_codec_par ==> p_codec_ctx,初始化相应成员 ret = avcodec_parameters_to_context(p_codec_ctx, p_codec_par); if (ret < 0) { is->rvc_hostapi->Debug(MEDIA_LOG_ERROR, "avcodec_parameters_to_context() failed %d.", ret); avcodec_close(p_codec_ctx); avcodec_free_context(&p_codec_ctx); return ret; } // 1.3.3 p_codec_ctx初始化:使用p_codec初始化p_codec_ctx,初始化完成 ret = avcodec_open2(p_codec_ctx, p_codec, NULL); if (ret < 0) { is->rvc_hostapi->Debug(MEDIA_LOG_ERROR, "avcodec_open2() failed %d.", ret); avcodec_close(p_codec_ctx); avcodec_free_context(&p_codec_ctx); return ret; } p_codec_ctx->pkt_timebase = is->m_paudio_stream[index]->time_base; is->m_pacodec_ctx[index] = p_codec_ctx; } // 2. 创建音频解码线程 is->m_audio_decode_tid = SDL_CreateThread(audio_decode_thread, "audio decode thread", is); if (NULL == is->m_audio_decode_tid) { is->rvc_hostapi->Debug(MEDIA_LOG_ERROR, "SDL_Create audio decode thread failed: %s.", SDL_GetError()); return -1; } else { ret = 0; is->rvc_hostapi->Debug(MEDIA_LOG_DEBUG, "create %s success, and thread id is %u.", SDL_GetThreadName(is->m_audio_decode_tid), SDL_GetThreadID(is->m_audio_decode_tid)); } return ret; } static int audio_resample(player_stat_t *is, int64_t audio_callback_time) { int data_size = 0, resampled_data_size = 0; int64_t dec_channel_layout = 0; av_unused double audio_clock0 = 0.0; int wanted_nb_samples = 0; frame_t *af = NULL; while (frame_queue_nb_remaining(&is->audio_frm_queue) == 0){ if ((av_gettime_relative() - audio_callback_time) > 1000000LL * is->audio_hw_buf_size / is->m_audio_param_tgt.bytes_per_sec / 2) { return -1; } av_usleep(1000); } // 若队列头部可读,则由af指向可读帧 if (!(af = frame_queue_peek_readable(&is->audio_frm_queue))) { return -1; } frame_queue_next(&is->audio_frm_queue); // 根据frame中指定的音频参数获取缓冲区的大小 data_size = av_samples_get_buffer_size(NULL, af->frame->channels, // 本行两参数:linesize,声道数 af->frame->nb_samples, // 本行一参数:本帧中包含的单个声道中的样本数 (AVSampleFormat)af->frame->format, 1); // 本行两参数:采样格式,不对齐 // 获取声道布局 dec_channel_layout = (af->frame->channel_layout && af->frame->channels == av_get_channel_layout_nb_channels(af->frame->channel_layout)) ? af->frame->channel_layout : av_get_default_channel_layout(af->frame->channels); wanted_nb_samples = af->frame->nb_samples; // is->audio_param_tgt是SDL可接受的音频帧数,是audio_open()中取得的参数 // 在audio_open()函数中又有“is->audio_src = is->audio_param_tgt” // 此处表示:如果frame中的音频参数 == is->audio_src == is->audio_param_tgt,那音频重采样的过程就免了(因此时is->swr_ctr是NULL) //      否则使用frame(源)和is->audio_param_tgt(目标)中的音频参数来设置is->swr_ctx,并使用frame中的音频参数来赋值is->audio_src if (af->frame->format != is->m_audio_param_src.fmt || dec_channel_layout != is->m_audio_param_src.channel_layout || af->frame->sample_rate != is->m_audio_param_src.freq) { swr_free(&is->m_paudio_swr_ctx); // 使用frame(源)和is->audio_param_tgt(目标)中的音频参数来设置is->audio_swr_ctx is->m_paudio_swr_ctx = swr_alloc_set_opts(NULL, is->m_audio_param_tgt.channel_layout, (AVSampleFormat)is->m_audio_param_tgt.fmt, is->m_audio_param_tgt.freq, dec_channel_layout, (AVSampleFormat)af->frame->format, af->frame->sample_rate, 0, NULL); if (!is->m_paudio_swr_ctx || swr_init(is->m_paudio_swr_ctx) < 0) { is->rvc_hostapi->Debug(MEDIA_LOG_ERROR, "Cannot create sample rate converter for conversion of %d Hz %s %d channels to %d Hz %s %d channels!", af->frame->sample_rate, av_get_sample_fmt_name((AVSampleFormat)af->frame->format), af->frame->channels, is->m_audio_param_tgt.freq, av_get_sample_fmt_name((AVSampleFormat)is->m_audio_param_tgt.fmt), is->m_audio_param_tgt.channels); swr_free(&is->m_paudio_swr_ctx); return -1; } // 使用frame中的参数更新is->audio_src,第一次更新后后面基本不用执行此if分支了,因为一个音频流中各frame通用参数一样 is->m_audio_param_src.channel_layout = dec_channel_layout; is->m_audio_param_src.channels = af->frame->channels; is->m_audio_param_src.freq = af->frame->sample_rate; is->m_audio_param_src.fmt = (AVSampleFormat)af->frame->format; } if (is->m_paudio_swr_ctx) { // 重采样输入参数1:输入音频样本数是af->frame->nb_samples // 重采样输入参数2:输入音频缓冲区 const uint8_t **in = (const uint8_t **)af->frame->extended_data; // 重采样输出参数1:输出音频缓冲区尺寸 // 重采样输出参数2:输出音频缓冲区 uint8_t **out = &is->m_paudio_frm_rwr; // 重采样输出参数:输出音频样本数(多加了256个样本) int out_count = (int64_t)wanted_nb_samples * is->m_audio_param_tgt.freq / af->frame->sample_rate + 256; // 重采样输出参数:输出音频缓冲区尺寸(以字节为单位) int out_size = av_samples_get_buffer_size(NULL, is->m_audio_param_tgt.channels, out_count, (AVSampleFormat)is->m_audio_param_tgt.fmt, 0); int len2 = 0; if (out_size < 0){ is->rvc_hostapi->Debug(MEDIA_LOG_ERROR, "av_samples_get_buffer_size() failed."); return -1; } av_fast_malloc(&is->m_paudio_frm_rwr, &is->audio_frm_rwr_size, out_size); if (!is->m_paudio_frm_rwr) { return AVERROR(ENOMEM); } // 音频重采样:返回值是重采样后得到的音频数据中单个声道的样本数 len2 = swr_convert(is->m_paudio_swr_ctx, out, out_count, in, af->frame->nb_samples); if (len2 < 0){ is->rvc_hostapi->Debug(MEDIA_LOG_ERROR, "swr_convert() failed."); return -1; } if (len2 == out_count){ is->rvc_hostapi->Debug(MEDIA_LOG_ERROR, "audio buffer is probably too small."); if (swr_init(is->m_paudio_swr_ctx) < 0) swr_free(&is->m_paudio_swr_ctx); } is->m_paudio_frm = is->m_paudio_frm_rwr; // 重采样返回的一帧音频数据大小(以字节为单位) resampled_data_size = len2 * is->m_audio_param_tgt.channels * av_get_bytes_per_sample((AVSampleFormat)is->m_audio_param_tgt.fmt); } else { // 未经重采样,则将指针指向frame中的音频数据 is->m_paudio_frm = af->frame->data[0]; resampled_data_size = data_size; } audio_clock0 = is->audio_clock; /* update the audio clock with the pts */ if (!isnan(af->pts)){ is->audio_clock = af->pts + (double)af->frame->nb_samples / af->frame->sample_rate; } else{ is->audio_clock = NAN; } is->audio_clock_serial = af->serial; #ifdef DEBUG { static double last_clock; last_clock = is->audio_clock; } #endif return resampled_data_size; } #ifdef _WIN32 static char* Utf8ToGB2312(const char* utf8) { int len = MultiByteToWideChar(CP_UTF8, 0, utf8, -1, NULL, 0); wchar_t* wstr = new wchar_t[len + 1]; memset(wstr, 0, len + 1); MultiByteToWideChar(CP_UTF8, 0, utf8, -1, wstr, len); len = WideCharToMultiByte(CP_ACP, 0, wstr, -1, NULL, 0, NULL, NULL); char* str = new char[len + 1]; memset(str, 0, len + 1); WideCharToMultiByte(CP_ACP, 0, wstr, -1, str, len, NULL, NULL); if (wstr) { delete []wstr; wstr = NULL; } return str; } #endif static int open_audio_playing(void *arg) { player_stat_t *is = (player_stat_t *)arg; SDL_AudioSpec wanted_spec = {0}; SDL_AudioSpec actual_spec = {0}; wanted_spec.freq = is->m_pacodec_ctx[is->m_iaudio_dec_index]->sample_rate; // 采样率 wanted_spec.format = AUDIO_S16SYS; // S表带符号,16是采样深度,SYS表采用系统字节序 wanted_spec.channels = is->m_pacodec_ctx[is->m_iaudio_dec_index]->channels; // 声音通道数 wanted_spec.silence = 0; // 静音值 // wanted_spec.samples = SDL_AUDIO_BUFFER_SIZE; // SDL声音缓冲区尺寸,单位是单声道采样点尺寸x通道数 // SDL声音缓冲区尺寸,单位是单声道采样点尺寸x声道数 wanted_spec.samples = FFMAX(SDL_AUDIO_MIN_BUFFER_SIZE, 2 << av_log2(wanted_spec.freq / SDL_AUDIO_MAX_CALLBACKS_PER_SEC)); wanted_spec.callback = sdl_audio_callback; // 回调函数,若为NULL,则应使用SDL_QueueAudio()机制 wanted_spec.userdata = is; // 提供给回调函数的参数 if (NULL == is->m_straudiodev) { int iaudioapeaker = SDL_GetNumAudioDevices(0); int i = 0; for (; i < iaudioapeaker; i++) { #ifdef _WIN32 char* strdevice = Utf8ToGB2312(SDL_GetAudioDeviceName(i, 0)); if (is->m_paudiodev && strstr(strdevice, is->m_paudiodev)) { const char* strdevname = SDL_GetAudioDeviceName(i, 0); is->m_straudiodev = av_strdup(strdevname); delete []strdevice; break; } else { delete[] strdevice; #else if (is->m_paudiodev && strstr(SDL_GetAudioDeviceName(i, 0), is->m_paudiodev)) { const char* strdevname = SDL_GetAudioDeviceName(i, 0); is->m_straudiodev = av_strdup(strdevname); break; #endif } } if (i == iaudioapeaker) { is->rvc_hostapi->Debug(MEDIA_LOG_ERROR, "matched audio device name (%s) failed!", is->m_straudiodev ? is->m_straudiodev : "null"); return -1; } } while (!(is->m_audio_dev = SDL_OpenAudioDevice(is->m_straudiodev, 0, &wanted_spec, &actual_spec, SDL_AUDIO_ALLOW_FREQUENCY_CHANGE | SDL_AUDIO_ALLOW_CHANNELS_CHANGE))){ is->rvc_hostapi->Debug(MEDIA_LOG_DEBUG, "SDL_OpenAudio (%d channels, %d Hz): %s",wanted_spec.channels, wanted_spec.freq, SDL_GetError()); if (!wanted_spec.channels) { if (!wanted_spec.freq) { is->rvc_hostapi->Debug(MEDIA_LOG_ERROR,"No more combinations to try, audio open failed!"); return -1; } } return -1; } is->m_audio_param_tgt.fmt = AV_SAMPLE_FMT_S16; is->m_audio_param_tgt.freq = actual_spec.freq; is->m_audio_param_tgt.channel_layout = av_get_default_channel_layout(actual_spec.channels); is->m_audio_param_tgt.channels = actual_spec.channels; is->m_audio_param_tgt.frame_size = av_samples_get_buffer_size(NULL, actual_spec.channels, 1, (AVSampleFormat)is->m_audio_param_tgt.fmt, 1); is->m_audio_param_tgt.bytes_per_sec = av_samples_get_buffer_size(NULL, actual_spec.channels, actual_spec.freq, (AVSampleFormat)is->m_audio_param_tgt.fmt, 1); if (is->m_audio_param_tgt.bytes_per_sec <= 0 || is->m_audio_param_tgt.frame_size <= 0){ is->rvc_hostapi->Debug(MEDIA_LOG_ERROR, "av_samples_get_buffer_size failed."); SDL_CloseAudioDevice(is->m_audio_dev); return -1; } is->m_audio_param_src = is->m_audio_param_tgt; is->audio_hw_buf_size = actual_spec.size; // SDL音频缓冲区大小 is->audio_frm_size = 0; is->audio_cp_index = 0; SDL_PauseAudioDevice(is->m_audio_dev, 0); return 0; } // 音频处理回调函数。读队列获取音频包,解码,播放 // 此函数被SDL按需调用,此函数不在用户主线程中,因此数据需要保护 // \param[in] opaque 用户在注册回调函数时指定的参数 // \param[out] stream 音频数据缓冲区地址,将解码后的音频数据填入此缓冲区 // \param[out] len 音频数据缓冲区大小,单位字节 // 回调函数返回后,stream指向的音频缓冲区将变为无效 // 双声道采样点的顺序为LRLRLR static void sdl_audio_callback(void *opaque, uint8_t*stream, int len) { player_stat_t *is = (player_stat_t *)opaque; int audio_size = 0, len1 = 0; int64_t audio_callback_time = av_gettime_relative(); while (len > 0 && false == is->buser_stop) // 输入参数len等于is->audio_hw_buf_size,是audio_open()中申请到的SDL音频缓冲区大小 { if (is->audio_cp_index >= (int)is->audio_frm_size){ // 1. 从音频frame队列中取出一个frame,转换为音频设备支持的格式,返回值是重采样音频帧的大小 audio_size = audio_resample(is, audio_callback_time); if (audio_size < 0){ if (-1 == audio_size) { if (is->m_baudio_decode_finished) { //is->rvc_hostapi->Debug(MEDIA_LOG_DEBUG, "audio_size is -1 sdl_audio_callback return, and set abort flag to true."); is->on_audio_play_finished(is->user_data); //is->rvc_hostapi->Debug(MEDIA_LOG_DEBUG, "SDL_SemPost m_audio_play_wait_sem."); SDL_SemPost(is->m_audio_play_wait_sem); return; } } /* if error, just output silence */ is->m_paudio_frm = NULL; is->audio_frm_size = SDL_AUDIO_MIN_BUFFER_SIZE / is->m_audio_param_tgt.frame_size * is->m_audio_param_tgt.frame_size; } else{ is->audio_frm_size = audio_size; } is->audio_cp_index = 0; } // 引入is->audio_cp_index的作用:防止一帧音频数据大小超过SDL音频缓冲区大小,这样一帧数据需要经过多次拷贝 // 用is->audio_cp_index标识重采样帧中已拷入SDL音频缓冲区的数据位置索引,len1表示本次拷贝的数据量 len1 = is->audio_frm_size - is->audio_cp_index; if (len1 > len){ len1 = len; } // 2. 将转换后的音频数据拷贝到音频缓冲区stream中,之后的播放就是音频设备驱动程序的工作了 if (is->m_paudio_frm != NULL){ SDL_memset(stream, 0, len1); int ivolume = is->uVolume; if (0 == is->on_audio_volume(&ivolume,is->user_data)){ //is->rvc_hostapi->Debug(MEDIA_LOG_DEBUG, "on_audio_volume success."); } //is->rvc_hostapi->Debug(MEDIA_LOG_DEBUG, "audio uVolume is %d.", ivolume); //SDL_MixAudio(stream, (uint8_t*)is->p_audio_frm + is->audio_cp_index, len1, ivolume); SDL_MixAudioFormat(stream, (uint8_t*)is->m_paudio_frm + is->audio_cp_index, AUDIO_S16SYS, len1, ivolume); //is->rvc_hostapi->Debug(MEDIA_LOG_DEBUG, "%s:%d SDL_MixAudioFormat audio length is %d.", __FUNCTION__, __LINE__, len1); if (is->m_prvc_cb && is->m_prvc_cb->cb_playing_audiodata) { is->m_prvc_cb->cb_playing_audiodata(&(is->m_audio_param_tgt), (uint8_t*)is->m_paudio_frm + is->audio_cp_index, len1, is->m_prvc_cb->user_data); } } else{ SDL_memset(stream, 0, len1); } len -= len1; stream += len1; is->audio_cp_index += len1; } // is->audio_write_buf_size是本帧中尚未拷入SDL音频缓冲区的数据量 is->audio_write_buf_size = is->audio_frm_size - is->audio_cp_index; //is->rvc_hostapi->Debug("audio_write_buf_size == %d.", is->audio_write_buf_size); /* Let's assume the audio driver that is used by SDL has two periods. */ // 3. 更新时钟 if (!isnan(is->audio_clock)) { // 更新音频时钟,更新时刻:每次往声卡缓冲区拷入数据后 // 前面audio_decode_frame中更新的is->audio_clock是以音频帧为单位,所以此处第二个参数要减去未拷贝数据量占用的时间 set_clock_at(&is->audio_clk, is->audio_clock - (double)(2 * is->audio_hw_buf_size + is->audio_write_buf_size) / is->m_audio_param_tgt.bytes_per_sec, is->audio_clock_serial, audio_callback_time / 1000000.0); } } int open_audio(player_stat_t *is) { int iret = -1; if (-1 == is->audio_idx[is->m_icurrent_index]){ is->rvc_hostapi->Debug(MEDIA_LOG_DEBUG, "not find audio stream"); } else { if (0 == open_audio_stream(is)) { iret = open_audio_playing(is); if (0 != iret) { is->rvc_hostapi->Debug(MEDIA_LOG_ERROR, "open audio playing failed"); } } else { is->rvc_hostapi->Debug(MEDIA_LOG_ERROR, "open audio stream failed"); } } return iret; }