myvtm/Other/libvideoframework/video_coding/frame_dropper.cpp

#include "../video_coding/frame_dropper.h"
#include "../video_coding/base/exp_filter.h"
#include "../video_coding/base/video_coding_log.h"

#include <algorithm>

namespace video_coding {

	static const float kDefaultFrameSizeAlpha = 0.9f;
	static const float kDefaultKeyFrameRatioAlpha = 0.99f;
	// 1 key frame every 10th second in 30 fps.
	static const float kDefaultKeyFrameRatioValue = 1 / 300.0f;

	static const float kDefaultDropRatioAlpha = 0.9f;
	static const float kDefaultDropRatioValue = 0.96f;
	// Maximum duration over which frames are continuously dropped.
	static const float kDefaultMaxDropDurationSecs = 4.0f;

	// Default target bitrate.
	// TODO(isheriff): Should this be higher to avoid dropping too many packets when
	// the bandwidth is unknown at the start ?
	static const float kDefaultTargetBitrateKbps = 300.0f;
	static const float kDefaultIncomingFrameRate = 30;
	static const float kLeakyBucketSizeSeconds = 0.5f;

	// A delta frame that is bigger than |kLargeDeltaFactor| times the average
	// delta frame is a large frame that is spread out for accumulation.
	static const int kLargeDeltaFactor = 3;

	// Cap on the frame size accumulator to prevent excessive drops.
	static const float kAccumulatorCapBufferSizeSecs = 3.0f;

	struct _FrameDropper {
		ExpFilter *key_frame_ratio_;
		ExpFilter *delta_frame_size_avg_kbits_;
		// Key frames and large delta frames are not immediately accumulated in the
		// bucket since they can immediately overflow the bucket leading to large
		// drops on the following packets that may be much smaller. Instead these
		// large frames are accumulated over several frames when the bucket leaks.

		// |large_frame_accumulation_spread_| represents the number of frames over
		// which a large frame is accumulated.
		float large_frame_accumulation_spread_;
		// |large_frame_accumulation_count_| represents the number of frames left
		// to finish accumulating a large frame.
		int large_frame_accumulation_count_;
		// |large_frame_accumulation_chunk_size_| represents the size of a single
		// chunk for large frame accumulation.
		float large_frame_accumulation_chunk_size_;

		float accumulator_;
		float accumulator_max_;
		float target_bitrate_;
		bool drop_next_;
		ExpFilter *drop_ratio_;
		int drop_count_;
		float incoming_frame_rate_;
		bool was_below_max_;
		bool enabled_;
		float max_drop_duration_secs_;
	};


	static void frame_dropper_update_ratio(FrameDropper *obj);
	static void frame_dropper_cap_accumulator(FrameDropper *obj);


	FrameDropper *frame_dropper_new() {
		FrameDropper *obj = (FrameDropper *)malloc(sizeof(FrameDropper));
		memset(obj, 0, sizeof(FrameDropper));

		obj->key_frame_ratio_ = exp_filter_new(kDefaultKeyFrameRatioAlpha);
		obj->delta_frame_size_avg_kbits_ = exp_filter_new(kDefaultFrameSizeAlpha);
		obj->drop_ratio_ = exp_filter_new(kDefaultDropRatioAlpha, kDefaultDropRatioValue);
		obj->enabled_ = true;
		obj->max_drop_duration_secs_ = kDefaultMaxDropDurationSecs;

		video_coding_debug(VIDEO_CODING_DEBUG_ENABLE, "frame_dropper_new key_frame_ratio: %f delta_frame_size_avg_kbits_ratio: %f drop_ratio: %f max_drop_duration_secs: %f.",
			kDefaultKeyFrameRatioAlpha, kDefaultFrameSizeAlpha, kDefaultDropRatioAlpha, kDefaultMaxDropDurationSecs);

		frame_dropper_reset(obj);
		return obj;
	}


	void frame_dropper_destroy(FrameDropper *obj) {
		if (obj != NULL) {
			exp_filter_destroy(obj->key_frame_ratio_);
			exp_filter_destroy(obj->delta_frame_size_avg_kbits_);
			exp_filter_destroy(obj->drop_ratio_);
			free(obj);

			video_coding_debug(VIDEO_CODING_DEBUG_ENABLE, "frame_dropper_destroy.");
		}
	}


	void frame_dropper_reset(FrameDropper *obj) {
		exp_filter_reset(obj->key_frame_ratio_, kDefaultKeyFrameRatioAlpha);
		exp_filter_apply(obj->key_frame_ratio_, 1.0f, kDefaultKeyFrameRatioValue);
		exp_filter_reset(obj->delta_frame_size_avg_kbits_, kDefaultFrameSizeAlpha);

		obj->accumulator_ = 0.0f;
		obj->accumulator_max_ = kDefaultTargetBitrateKbps / 2;
		obj->target_bitrate_ = kDefaultTargetBitrateKbps;
		obj->incoming_frame_rate_ = kDefaultIncomingFrameRate;

		obj->large_frame_accumulation_count_ = 0;
		obj->large_frame_accumulation_chunk_size_ = 0;
		obj->large_frame_accumulation_spread_ = 0.5 * kDefaultIncomingFrameRate;

		obj->drop_next_ = false;
		exp_filter_reset(obj->drop_ratio_, 0.9f);
		exp_filter_apply(obj->drop_ratio_, 0.0f, 0.0f);
		obj->drop_count_ = 0;
		obj->was_below_max_ = true;

		video_coding_debug(VIDEO_CODING_DEBUG_ENABLE, "frame_dropper_reset.");
	}

	void frame_dropper_enable(FrameDropper *obj, bool enable) {
		obj->enabled_ = enable;
		video_coding_debug(VIDEO_CODING_DEBUG_FRAME_DROPPER, "frame_dropper_enable enable: %d.", enable);
	}

	//deltaFrame : 0:key frame 1:P frame
	//编码后,帧大小累加到accumulator_中
	void frame_dropper_fill(FrameDropper *obj, size_t framesize_bytes, bool delta_frame) {
		if (!obj->enabled_) {
			return;
		}
		float framesize_kbits = 8.0f * static_cast<float>(framesize_bytes) / 1000.0f;
		if (!delta_frame) {
			exp_filter_apply(obj->key_frame_ratio_, 1.0, 1.0);
			// Do not spread if we are already doing it (or we risk dropping bits that
			// need accumulation). Given we compute the key frame ratio and spread
			// based on that, this should not normally happen.
			if (obj->large_frame_accumulation_count_ == 0) {
				if (exp_filter_filtered(obj->key_frame_ratio_) > 1e-5 &&
					1 / exp_filter_filtered(obj->key_frame_ratio_) < obj->large_frame_accumulation_spread_) {
					obj->large_frame_accumulation_count_ =
						static_cast<int32_t>(1 / exp_filter_filtered(obj->key_frame_ratio_) + 0.5);
				}
				else {
					obj->large_frame_accumulation_count_ =
						static_cast<int32_t>(obj->large_frame_accumulation_spread_ + 0.5);
				}
				obj->large_frame_accumulation_chunk_size_ =
					framesize_kbits / obj->large_frame_accumulation_count_;
				framesize_kbits = 0;
			}
		}
		else {
			// Identify if it is an unusually large delta frame and spread accumulation
			// if that is the case.  // >3*delta_frame_size_avg_kbits_的帧不参与accumulator_计算
			if (exp_filter_filtered(obj->delta_frame_size_avg_kbits_) != -1 &&
				(framesize_kbits >
					kLargeDeltaFactor * exp_filter_filtered(obj->delta_frame_size_avg_kbits_)) &&
				obj->large_frame_accumulation_count_ == 0) {
				obj->large_frame_accumulation_count_ =
					static_cast<int32_t>(obj->large_frame_accumulation_spread_ + 0.5);
				obj->large_frame_accumulation_chunk_size_ =
					framesize_kbits / obj->large_frame_accumulation_count_;
				framesize_kbits = 0;
			}
			else {
				exp_filter_apply(obj->delta_frame_size_avg_kbits_, 1, framesize_kbits);
			}
			exp_filter_apply(obj->key_frame_ratio_, 1.0, 0.0);
		}
		// Change the level of the accumulator (bucket)
		obj->accumulator_ += framesize_kbits;

		video_coding_debug(VIDEO_CODING_DEBUG_FRAME_DROPPER, "frame_dropper_fill accumulator: %f.", obj->accumulator_);
		frame_dropper_cap_accumulator(obj);
	}

	//编码前accumulator_-平均帧大小
	//根据输入帧率计算每帧大小，从accumulator_减去此平均帧大小
	void frame_dropper_leak(FrameDropper *obj, uint32_t input_framerate) {
		if (!obj->enabled_) {
			return;
		}
		if (input_framerate < 1) {
			return;
		}
		if (obj->target_bitrate_ < 0.0f) {
			return;
		}
		// Add lower bound for large frame accumulation spread.
		obj->large_frame_accumulation_spread_ = std::max<float>(0.5 * input_framerate, 5.0);
		// Expected bits per frame based on current input frame rate.
		float expected_bits_per_frame = obj->target_bitrate_ / input_framerate;
		if (obj->large_frame_accumulation_count_ > 0) {
			expected_bits_per_frame -= obj->large_frame_accumulation_chunk_size_;
			--obj->large_frame_accumulation_count_;
		}
		obj->accumulator_ -= expected_bits_per_frame;
		if (obj->accumulator_ < 0.0f) {
			obj->accumulator_ = 0.0f;
		}

		video_coding_debug(VIDEO_CODING_DEBUG_FRAME_DROPPER, "frame_dropper_leak accumulator: %f input_framerate: %u.",
			obj->accumulator_, input_framerate);
		frame_dropper_update_ratio(obj);
	}

	//accumulator_超过accumulator_max_，丢帧，否则不丢帧
	static void frame_dropper_update_ratio(FrameDropper *obj) {
		if (obj->accumulator_ > 1.3f * obj->accumulator_max_) {
			// Too far above accumulator max, react faster.
			exp_filter_update_base(obj->drop_ratio_, 0.8f);
		}
		else {
			// Go back to normal reaction.
			exp_filter_update_base(obj->drop_ratio_, 0.9f);
		}
		if (obj->accumulator_ > obj->accumulator_max_) {
			// We are above accumulator max, and should ideally drop a frame. Increase
			// the drop_ratio_ and drop the frame later.
			if (obj->was_below_max_) {
				obj->drop_next_ = true; //丢掉下一帧
			}
			exp_filter_apply(obj->drop_ratio_, 1.0f, 1.0f); //因为丢帧，所以sample为1
			exp_filter_update_base(obj->drop_ratio_, 0.9f);
		}
		else {
			exp_filter_apply(obj->drop_ratio_, 1.0f, 0.0f); //不丢帧，sample为0
		}
		obj->was_below_max_ = obj->accumulator_ < obj->accumulator_max_;

		video_coding_debug(VIDEO_CODING_DEBUG_FRAME_DROPPER, "frame_dropper_update_ratio accumulator: %f drop_ratio_: %f.",
			obj->accumulator_, exp_filter_filtered(obj->drop_ratio_));
	}

	// This function signals when to drop frames to the caller. It makes use of the
	// drop_ratio_ to smooth out the drops over time.
	bool frame_dropper_drop_frame(FrameDropper *obj) {
		if (!obj->enabled_) {
			return false;
		}
		if (obj->drop_next_) {
			obj->drop_next_ = false;
			obj->drop_count_ = 0;
		}

		video_coding_debug(VIDEO_CODING_DEBUG_FRAME_DROPPER, "frame_dropper_leak drop_ratio_: %f drop_count: %d.",
			exp_filter_filtered(obj->drop_ratio_), obj->drop_count_);

		if (exp_filter_filtered(obj->drop_ratio_) >= 0.5f) {  // Drops per keep //每隔若干帧丢limit帧  P***P其中limit为3，p为保留，*为丢
		  // Limit is the number of frames we should drop between each kept frame
		  // to keep our drop ratio. limit is positive in this case.
			float denom = 1.0f - exp_filter_filtered(obj->drop_ratio_);
			if (denom < 1e-5) {
				denom = 1e-5f;
			}
			int32_t limit = static_cast<int32_t>(1.0f / denom - 1.0f + 0.5f);
			// Put a bound on the max amount of dropped frames between each kept
			// frame, in terms of frame rate and window size (secs).
			int max_limit =
				static_cast<int>(obj->incoming_frame_rate_ * obj->max_drop_duration_secs_);
			if (limit > max_limit) {
				limit = max_limit;
			}
			if (obj->drop_count_ < 0) {
				// Reset the drop_count_ since it was negative and should be positive.
				obj->drop_count_ = -obj->drop_count_;
			}
			if (obj->drop_count_ < limit) {
				// As long we are below the limit we should drop frames.
				obj->drop_count_++;
				return true;
			}
			else {
				// Only when we reset drop_count_ a frame should be kept.
				obj->drop_count_ = 0;
				return false;
			}
		}
		else if (exp_filter_filtered(obj->drop_ratio_) > 0.0f &&
			exp_filter_filtered(obj->drop_ratio_) < 0.5f) {  // Keeps per drop //每隔若干帧保留limit帧  *pppp*其中limit为4，p为保留，*为丢
   // Limit is the number of frames we should keep between each drop
   // in order to keep the drop ratio. limit is negative in this case,
   // and the drop_count_ is also negative.
			float denom = exp_filter_filtered(obj->drop_ratio_);
			if (denom < 1e-5) {
				denom = 1e-5f;
			}
			int32_t limit = -static_cast<int32_t>(1.0f / denom - 1.0f + 0.5f);
			if (obj->drop_count_ > 0) {
				// Reset the drop_count_ since we have a positive
				// drop_count_, and it should be negative.
				obj->drop_count_ = -obj->drop_count_;
			}
			if (obj->drop_count_ > limit) {
				if (obj->drop_count_ == 0) {
					// Drop frames when we reset drop_count_.
					obj->drop_count_--;
					return true;
				}
				else {
					// Keep frames as long as we haven't reached limit.
					obj->drop_count_--;
					return false;
				}
			}
			else {
				obj->drop_count_ = 0;
				return false;
			}
		}
		obj->drop_count_ = 0;
		return false;
	}

	void frame_dropper_set_rates(FrameDropper *obj, float bitrate, float incoming_frame_rate) {
		// Bit rate of -1 means infinite bandwidth.
		obj->accumulator_max_ = bitrate * kLeakyBucketSizeSeconds;
		if (obj->target_bitrate_ > 0.0f && bitrate < obj->target_bitrate_ &&
			obj->accumulator_ > obj->accumulator_max_) {
			// Rescale the accumulator level if the accumulator max decreases
			obj->accumulator_ = bitrate / obj->target_bitrate_ * obj->accumulator_;
		}
		obj->target_bitrate_ = bitrate;
		frame_dropper_cap_accumulator(obj);
		obj->incoming_frame_rate_ = incoming_frame_rate;

		video_coding_debug(VIDEO_CODING_DEBUG_FRAME_DROPPER, "frame_dropper_set_rates bitrate: %f incoming_frame_rate: %f.",
			bitrate, incoming_frame_rate);
	}

	// Put a cap on the accumulator, i.e., don't let it grow beyond some level.
	// This is a temporary fix for screencasting where very large frames from
	// encoder will cause very slow response (too many frame drops).
	// TODO(isheriff): Remove this now that large delta frames are also spread out ?
	static void frame_dropper_cap_accumulator(FrameDropper *obj) {
		float max_accumulator = obj->target_bitrate_ * kAccumulatorCapBufferSizeSecs;
		if (obj->accumulator_ > max_accumulator) {
			obj->accumulator_ = max_accumulator;
		}
	}

}