niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 1 | /* |
marpan@webrtc.org | 9d76b4e | 2012-02-28 23:39:31 +0000 | [diff] [blame] | 2 | * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Henrik Kjellander | 2557b86 | 2015-11-18 22:00:21 +0100 | [diff] [blame] | 11 | #include "webrtc/modules/video_coding/media_optimization.h" |
stefan@webrtc.org | e0d6fa4 | 2012-03-20 22:10:56 +0000 | [diff] [blame] | 12 | |
pbos | 854e84c | 2015-11-16 16:39:06 -0800 | [diff] [blame] | 13 | #include "webrtc/base/logging.h" |
kjellander@webrtc.org | b7ce964 | 2015-11-18 23:04:10 +0100 | [diff] [blame] | 14 | #include "webrtc/modules/video_coding/utility/frame_dropper.h" |
Henrik Kjellander | 98f5351 | 2015-10-28 18:17:40 +0100 | [diff] [blame] | 15 | #include "webrtc/system_wrappers/include/clock.h" |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 16 | |
| 17 | namespace webrtc { |
stefan@webrtc.org | a64300a | 2013-03-04 15:24:40 +0000 | [diff] [blame] | 18 | namespace media_optimization { |
andresp@webrtc.org | e682aa5 | 2013-12-19 10:59:48 +0000 | [diff] [blame] | 19 | namespace { |
| 20 | void UpdateProtectionCallback( |
| 21 | VCMProtectionMethod* selected_method, |
| 22 | uint32_t* video_rate_bps, |
| 23 | uint32_t* nack_overhead_rate_bps, |
| 24 | uint32_t* fec_overhead_rate_bps, |
| 25 | VCMProtectionCallback* video_protection_callback) { |
| 26 | FecProtectionParams delta_fec_params; |
| 27 | FecProtectionParams key_fec_params; |
| 28 | // Get the FEC code rate for Key frames (set to 0 when NA). |
| 29 | key_fec_params.fec_rate = selected_method->RequiredProtectionFactorK(); |
| 30 | |
| 31 | // Get the FEC code rate for Delta frames (set to 0 when NA). |
| 32 | delta_fec_params.fec_rate = selected_method->RequiredProtectionFactorD(); |
| 33 | |
andresp@webrtc.org | e682aa5 | 2013-12-19 10:59:48 +0000 | [diff] [blame] | 34 | // The RTP module currently requires the same |max_fec_frames| for both |
| 35 | // key and delta frames. |
| 36 | delta_fec_params.max_fec_frames = selected_method->MaxFramesFec(); |
| 37 | key_fec_params.max_fec_frames = selected_method->MaxFramesFec(); |
| 38 | |
| 39 | // Set the FEC packet mask type. |kFecMaskBursty| is more effective for |
| 40 | // consecutive losses and little/no packet re-ordering. As we currently |
| 41 | // do not have feedback data on the degree of correlated losses and packet |
| 42 | // re-ordering, we keep default setting to |kFecMaskRandom| for now. |
| 43 | delta_fec_params.fec_mask_type = kFecMaskRandom; |
| 44 | key_fec_params.fec_mask_type = kFecMaskRandom; |
| 45 | |
| 46 | // TODO(Marco): Pass FEC protection values per layer. |
philipel | 9d3ab61 | 2015-12-21 04:12:39 -0800 | [diff] [blame] | 47 | video_protection_callback->ProtectionRequest( |
| 48 | &delta_fec_params, &key_fec_params, video_rate_bps, |
| 49 | nack_overhead_rate_bps, fec_overhead_rate_bps); |
andresp@webrtc.org | e682aa5 | 2013-12-19 10:59:48 +0000 | [diff] [blame] | 50 | } |
| 51 | } // namespace |
| 52 | |
| 53 | struct MediaOptimization::EncodedFrameSample { |
pkasting@chromium.org | 4591fbd | 2014-11-20 22:28:14 +0000 | [diff] [blame] | 54 | EncodedFrameSample(size_t size_bytes, |
andresp@webrtc.org | e682aa5 | 2013-12-19 10:59:48 +0000 | [diff] [blame] | 55 | uint32_t timestamp, |
| 56 | int64_t time_complete_ms) |
| 57 | : size_bytes(size_bytes), |
| 58 | timestamp(timestamp), |
| 59 | time_complete_ms(time_complete_ms) {} |
| 60 | |
pkasting@chromium.org | 4591fbd | 2014-11-20 22:28:14 +0000 | [diff] [blame] | 61 | size_t size_bytes; |
andresp@webrtc.org | e682aa5 | 2013-12-19 10:59:48 +0000 | [diff] [blame] | 62 | uint32_t timestamp; |
| 63 | int64_t time_complete_ms; |
| 64 | }; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 65 | |
stefan@webrtc.org | 34c5da6 | 2014-04-11 14:08:35 +0000 | [diff] [blame] | 66 | MediaOptimization::MediaOptimization(Clock* clock) |
wuchengli@chromium.org | ae7cfd7 | 2014-06-30 08:01:47 +0000 | [diff] [blame] | 67 | : crit_sect_(CriticalSectionWrapper::CreateCriticalSection()), |
| 68 | clock_(clock), |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 69 | max_bit_rate_(0), |
| 70 | send_codec_type_(kVideoCodecUnknown), |
| 71 | codec_width_(0), |
| 72 | codec_height_(0), |
| 73 | user_frame_rate_(0), |
henrik.lundin@webrtc.org | b426c46 | 2013-09-24 07:41:53 +0000 | [diff] [blame] | 74 | frame_dropper_(new FrameDropper), |
| 75 | loss_prot_logic_( |
| 76 | new VCMLossProtectionLogic(clock_->TimeInMilliseconds())), |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 77 | fraction_lost_(0), |
| 78 | send_statistics_zero_encode_(0), |
| 79 | max_payload_size_(1460), |
pbos | 7367463 | 2015-10-29 15:45:00 -0700 | [diff] [blame] | 80 | video_target_bitrate_(0), |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 81 | incoming_frame_rate_(0), |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 82 | encoded_frame_samples_(), |
| 83 | avg_sent_bit_rate_bps_(0), |
| 84 | avg_sent_framerate_(0), |
| 85 | key_frame_cnt_(0), |
| 86 | delta_frame_cnt_(0), |
henrik.lundin@webrtc.org | 544b17c | 2013-09-26 12:05:15 +0000 | [diff] [blame] | 87 | num_layers_(0), |
henrik.lundin@webrtc.org | ce8e093 | 2013-11-18 12:18:43 +0000 | [diff] [blame] | 88 | suspension_enabled_(false), |
| 89 | video_suspended_(false), |
| 90 | suspension_threshold_bps_(0), |
| 91 | suspension_window_bps_(0) { |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 92 | memset(send_statistics_, 0, sizeof(send_statistics_)); |
| 93 | memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_)); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 94 | } |
| 95 | |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 96 | MediaOptimization::~MediaOptimization(void) { |
| 97 | loss_prot_logic_->Release(); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 98 | } |
| 99 | |
andresp@webrtc.org | e682aa5 | 2013-12-19 10:59:48 +0000 | [diff] [blame] | 100 | void MediaOptimization::Reset() { |
wuchengli@chromium.org | ae7cfd7 | 2014-06-30 08:01:47 +0000 | [diff] [blame] | 101 | CriticalSectionScoped lock(crit_sect_.get()); |
philipel | 9d3ab61 | 2015-12-21 04:12:39 -0800 | [diff] [blame] | 102 | SetEncodingDataInternal(kVideoCodecUnknown, 0, 0, 0, 0, 0, 0, |
| 103 | max_payload_size_); |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 104 | memset(incoming_frame_times_, -1, sizeof(incoming_frame_times_)); |
| 105 | incoming_frame_rate_ = 0.0; |
| 106 | frame_dropper_->Reset(); |
| 107 | loss_prot_logic_->Reset(clock_->TimeInMilliseconds()); |
| 108 | frame_dropper_->SetRates(0, 0); |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 109 | loss_prot_logic_->UpdateFrameRate(incoming_frame_rate_); |
| 110 | loss_prot_logic_->Reset(clock_->TimeInMilliseconds()); |
| 111 | send_statistics_zero_encode_ = 0; |
pbos | 7367463 | 2015-10-29 15:45:00 -0700 | [diff] [blame] | 112 | video_target_bitrate_ = 0; |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 113 | codec_width_ = 0; |
| 114 | codec_height_ = 0; |
| 115 | user_frame_rate_ = 0; |
| 116 | key_frame_cnt_ = 0; |
| 117 | delta_frame_cnt_ = 0; |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 118 | encoded_frame_samples_.clear(); |
| 119 | avg_sent_bit_rate_bps_ = 0; |
| 120 | num_layers_ = 1; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 121 | } |
| 122 | |
andresp@webrtc.org | e682aa5 | 2013-12-19 10:59:48 +0000 | [diff] [blame] | 123 | void MediaOptimization::SetEncodingData(VideoCodecType send_codec_type, |
| 124 | int32_t max_bit_rate, |
andresp@webrtc.org | e682aa5 | 2013-12-19 10:59:48 +0000 | [diff] [blame] | 125 | uint32_t target_bitrate, |
| 126 | uint16_t width, |
| 127 | uint16_t height, |
Peter Boström | df66453 | 2015-05-12 12:22:14 +0200 | [diff] [blame] | 128 | uint32_t frame_rate, |
andresp@webrtc.org | e682aa5 | 2013-12-19 10:59:48 +0000 | [diff] [blame] | 129 | int num_layers, |
| 130 | int32_t mtu) { |
wuchengli@chromium.org | ae7cfd7 | 2014-06-30 08:01:47 +0000 | [diff] [blame] | 131 | CriticalSectionScoped lock(crit_sect_.get()); |
philipel | 9d3ab61 | 2015-12-21 04:12:39 -0800 | [diff] [blame] | 132 | SetEncodingDataInternal(send_codec_type, max_bit_rate, frame_rate, |
| 133 | target_bitrate, width, height, num_layers, mtu); |
wuchengli@chromium.org | ae7cfd7 | 2014-06-30 08:01:47 +0000 | [diff] [blame] | 134 | } |
| 135 | |
| 136 | void MediaOptimization::SetEncodingDataInternal(VideoCodecType send_codec_type, |
| 137 | int32_t max_bit_rate, |
| 138 | uint32_t frame_rate, |
| 139 | uint32_t target_bitrate, |
| 140 | uint16_t width, |
| 141 | uint16_t height, |
| 142 | int num_layers, |
| 143 | int32_t mtu) { |
andresp@webrtc.org | e682aa5 | 2013-12-19 10:59:48 +0000 | [diff] [blame] | 144 | // Everything codec specific should be reset here since this means the codec |
Peter Boström | ad6fc5a | 2016-05-12 03:01:31 +0200 | [diff] [blame^] | 145 | // has changed. |
andresp@webrtc.org | e682aa5 | 2013-12-19 10:59:48 +0000 | [diff] [blame] | 146 | |
| 147 | max_bit_rate_ = max_bit_rate; |
| 148 | send_codec_type_ = send_codec_type; |
pbos | 7367463 | 2015-10-29 15:45:00 -0700 | [diff] [blame] | 149 | video_target_bitrate_ = target_bitrate; |
andresp@webrtc.org | e682aa5 | 2013-12-19 10:59:48 +0000 | [diff] [blame] | 150 | float target_bitrate_kbps = static_cast<float>(target_bitrate) / 1000.0f; |
| 151 | loss_prot_logic_->UpdateBitRate(target_bitrate_kbps); |
| 152 | loss_prot_logic_->UpdateFrameRate(static_cast<float>(frame_rate)); |
| 153 | loss_prot_logic_->UpdateFrameSize(width, height); |
| 154 | loss_prot_logic_->UpdateNumLayers(num_layers); |
| 155 | frame_dropper_->Reset(); |
| 156 | frame_dropper_->SetRates(target_bitrate_kbps, static_cast<float>(frame_rate)); |
| 157 | user_frame_rate_ = static_cast<float>(frame_rate); |
| 158 | codec_width_ = width; |
| 159 | codec_height_ = height; |
| 160 | num_layers_ = (num_layers <= 1) ? 1 : num_layers; // Can also be zero. |
| 161 | max_payload_size_ = mtu; |
andresp@webrtc.org | e682aa5 | 2013-12-19 10:59:48 +0000 | [diff] [blame] | 162 | } |
| 163 | |
| 164 | uint32_t MediaOptimization::SetTargetRates( |
| 165 | uint32_t target_bitrate, |
| 166 | uint8_t fraction_lost, |
pkasting@chromium.org | 16825b1 | 2015-01-12 21:51:21 +0000 | [diff] [blame] | 167 | int64_t round_trip_time_ms, |
Peter Boström | ad6fc5a | 2016-05-12 03:01:31 +0200 | [diff] [blame^] | 168 | VCMProtectionCallback* protection_callback) { |
wuchengli@chromium.org | ae7cfd7 | 2014-06-30 08:01:47 +0000 | [diff] [blame] | 169 | CriticalSectionScoped lock(crit_sect_.get()); |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 170 | VCMProtectionMethod* selected_method = loss_prot_logic_->SelectedMethod(); |
| 171 | float target_bitrate_kbps = static_cast<float>(target_bitrate) / 1000.0f; |
| 172 | loss_prot_logic_->UpdateBitRate(target_bitrate_kbps); |
| 173 | loss_prot_logic_->UpdateRtt(round_trip_time_ms); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 174 | |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 175 | // Get frame rate for encoder: this is the actual/sent frame rate. |
wuchengli@chromium.org | ae7cfd7 | 2014-06-30 08:01:47 +0000 | [diff] [blame] | 176 | float actual_frame_rate = SentFrameRateInternal(); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 177 | |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 178 | // Sanity check. |
| 179 | if (actual_frame_rate < 1.0) { |
| 180 | actual_frame_rate = 1.0; |
| 181 | } |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 182 | |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 183 | // Update frame rate for the loss protection logic class: frame rate should |
| 184 | // be the actual/sent rate. |
| 185 | loss_prot_logic_->UpdateFrameRate(actual_frame_rate); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 186 | |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 187 | fraction_lost_ = fraction_lost; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 188 | |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 189 | // Returns the filtered packet loss, used for the protection setting. |
| 190 | // The filtered loss may be the received loss (no filter), or some |
| 191 | // filtered value (average or max window filter). |
| 192 | // Use max window filter for now. |
| 193 | FilterPacketLossMode filter_mode = kMaxFilter; |
| 194 | uint8_t packet_loss_enc = loss_prot_logic_->FilteredLoss( |
| 195 | clock_->TimeInMilliseconds(), filter_mode, fraction_lost); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 196 | |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 197 | // For now use the filtered loss for computing the robustness settings. |
| 198 | loss_prot_logic_->UpdateFilteredLossPr(packet_loss_enc); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 199 | |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 200 | // Rate cost of the protection methods. |
pbos | 7367463 | 2015-10-29 15:45:00 -0700 | [diff] [blame] | 201 | float protection_overhead_rate = 0.0f; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 202 | |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 203 | // Update protection settings, when applicable. |
mflodman | fcf54bd | 2015-04-14 21:28:08 +0200 | [diff] [blame] | 204 | if (loss_prot_logic_->SelectedType() != kNone) { |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 205 | // Update method will compute the robustness settings for the given |
| 206 | // protection method and the overhead cost |
| 207 | // the protection method is set by the user via SetVideoProtection. |
| 208 | loss_prot_logic_->UpdateMethod(); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 209 | |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 210 | // Update protection callback with protection settings. |
| 211 | uint32_t sent_video_rate_bps = 0; |
| 212 | uint32_t sent_nack_rate_bps = 0; |
| 213 | uint32_t sent_fec_rate_bps = 0; |
| 214 | // Get the bit cost of protection method, based on the amount of |
| 215 | // overhead data actually transmitted (including headers) the last |
| 216 | // second. |
andresp@webrtc.org | e682aa5 | 2013-12-19 10:59:48 +0000 | [diff] [blame] | 217 | if (protection_callback) { |
philipel | 9d3ab61 | 2015-12-21 04:12:39 -0800 | [diff] [blame] | 218 | UpdateProtectionCallback(selected_method, &sent_video_rate_bps, |
| 219 | &sent_nack_rate_bps, &sent_fec_rate_bps, |
andresp@webrtc.org | e682aa5 | 2013-12-19 10:59:48 +0000 | [diff] [blame] | 220 | protection_callback); |
| 221 | } |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 222 | uint32_t sent_total_rate_bps = |
| 223 | sent_video_rate_bps + sent_nack_rate_bps + sent_fec_rate_bps; |
| 224 | // Estimate the overhead costs of the next second as staying the same |
| 225 | // wrt the source bitrate. |
| 226 | if (sent_total_rate_bps > 0) { |
pbos | 7367463 | 2015-10-29 15:45:00 -0700 | [diff] [blame] | 227 | protection_overhead_rate = |
| 228 | static_cast<float>(sent_nack_rate_bps + sent_fec_rate_bps) / |
| 229 | sent_total_rate_bps; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 230 | } |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 231 | // Cap the overhead estimate to 50%. |
pbos | 7367463 | 2015-10-29 15:45:00 -0700 | [diff] [blame] | 232 | if (protection_overhead_rate > 0.5) |
| 233 | protection_overhead_rate = 0.5; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 234 | |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 235 | // Get the effective packet loss for encoder ER when applicable. Should be |
| 236 | // passed to encoder via fraction_lost. |
| 237 | packet_loss_enc = selected_method->RequiredPacketLossER(); |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 238 | } |
stefan@webrtc.org | f4c8286 | 2011-12-13 15:38:14 +0000 | [diff] [blame] | 239 | |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 240 | // Source coding rate: total rate - protection overhead. |
pbos | 7367463 | 2015-10-29 15:45:00 -0700 | [diff] [blame] | 241 | video_target_bitrate_ = target_bitrate * (1.0 - protection_overhead_rate); |
| 242 | |
| 243 | // Cap target video bitrate to codec maximum. |
| 244 | if (max_bit_rate_ > 0 && video_target_bitrate_ > max_bit_rate_) { |
| 245 | video_target_bitrate_ = max_bit_rate_; |
| 246 | } |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 247 | |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 248 | // Update encoding rates following protection settings. |
| 249 | float target_video_bitrate_kbps = |
pbos | 7367463 | 2015-10-29 15:45:00 -0700 | [diff] [blame] | 250 | static_cast<float>(video_target_bitrate_) / 1000.0f; |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 251 | frame_dropper_->SetRates(target_video_bitrate_kbps, incoming_frame_rate_); |
| 252 | |
henrik.lundin@webrtc.org | ce8e093 | 2013-11-18 12:18:43 +0000 | [diff] [blame] | 253 | CheckSuspendConditions(); |
henrik.lundin@webrtc.org | 544b17c | 2013-09-26 12:05:15 +0000 | [diff] [blame] | 254 | |
pbos | 7367463 | 2015-10-29 15:45:00 -0700 | [diff] [blame] | 255 | return video_target_bitrate_; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 256 | } |
| 257 | |
pbos | ba8c15b | 2015-07-14 09:36:34 -0700 | [diff] [blame] | 258 | void MediaOptimization::SetProtectionMethod(VCMProtectionMethodEnum method) { |
wuchengli@chromium.org | ae7cfd7 | 2014-06-30 08:01:47 +0000 | [diff] [blame] | 259 | CriticalSectionScoped lock(crit_sect_.get()); |
pbos@webrtc.org | cade82c | 2015-03-12 10:39:24 +0000 | [diff] [blame] | 260 | loss_prot_logic_->SetMethod(method); |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 261 | } |
| 262 | |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 263 | uint32_t MediaOptimization::InputFrameRate() { |
wuchengli@chromium.org | ae7cfd7 | 2014-06-30 08:01:47 +0000 | [diff] [blame] | 264 | CriticalSectionScoped lock(crit_sect_.get()); |
| 265 | return InputFrameRateInternal(); |
| 266 | } |
| 267 | |
| 268 | uint32_t MediaOptimization::InputFrameRateInternal() { |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 269 | ProcessIncomingFrameRate(clock_->TimeInMilliseconds()); |
| 270 | return uint32_t(incoming_frame_rate_ + 0.5f); |
| 271 | } |
| 272 | |
| 273 | uint32_t MediaOptimization::SentFrameRate() { |
wuchengli@chromium.org | ae7cfd7 | 2014-06-30 08:01:47 +0000 | [diff] [blame] | 274 | CriticalSectionScoped lock(crit_sect_.get()); |
| 275 | return SentFrameRateInternal(); |
| 276 | } |
| 277 | |
| 278 | uint32_t MediaOptimization::SentFrameRateInternal() { |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 279 | PurgeOldFrameSamples(clock_->TimeInMilliseconds()); |
| 280 | UpdateSentFramerate(); |
| 281 | return avg_sent_framerate_; |
| 282 | } |
| 283 | |
| 284 | uint32_t MediaOptimization::SentBitRate() { |
wuchengli@chromium.org | ae7cfd7 | 2014-06-30 08:01:47 +0000 | [diff] [blame] | 285 | CriticalSectionScoped lock(crit_sect_.get()); |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 286 | const int64_t now_ms = clock_->TimeInMilliseconds(); |
| 287 | PurgeOldFrameSamples(now_ms); |
| 288 | UpdateSentBitrate(now_ms); |
| 289 | return avg_sent_bit_rate_bps_; |
| 290 | } |
| 291 | |
pbos@webrtc.org | 273a414 | 2014-12-01 15:23:21 +0000 | [diff] [blame] | 292 | int32_t MediaOptimization::UpdateWithEncodedData( |
| 293 | const EncodedImage& encoded_image) { |
| 294 | size_t encoded_length = encoded_image._length; |
| 295 | uint32_t timestamp = encoded_image._timeStamp; |
wuchengli@chromium.org | ae7cfd7 | 2014-06-30 08:01:47 +0000 | [diff] [blame] | 296 | CriticalSectionScoped lock(crit_sect_.get()); |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 297 | const int64_t now_ms = clock_->TimeInMilliseconds(); |
| 298 | PurgeOldFrameSamples(now_ms); |
| 299 | if (encoded_frame_samples_.size() > 0 && |
| 300 | encoded_frame_samples_.back().timestamp == timestamp) { |
| 301 | // Frames having the same timestamp are generated from the same input |
| 302 | // frame. We don't want to double count them, but only increment the |
| 303 | // size_bytes. |
| 304 | encoded_frame_samples_.back().size_bytes += encoded_length; |
| 305 | encoded_frame_samples_.back().time_complete_ms = now_ms; |
| 306 | } else { |
| 307 | encoded_frame_samples_.push_back( |
| 308 | EncodedFrameSample(encoded_length, timestamp, now_ms)); |
| 309 | } |
| 310 | UpdateSentBitrate(now_ms); |
| 311 | UpdateSentFramerate(); |
| 312 | if (encoded_length > 0) { |
Peter Boström | 49e196a | 2015-10-23 15:58:18 +0200 | [diff] [blame] | 313 | const bool delta_frame = encoded_image._frameType != kVideoFrameKey; |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 314 | |
| 315 | frame_dropper_->Fill(encoded_length, delta_frame); |
| 316 | if (max_payload_size_ > 0 && encoded_length > 0) { |
| 317 | const float min_packets_per_frame = |
| 318 | encoded_length / static_cast<float>(max_payload_size_); |
| 319 | if (delta_frame) { |
| 320 | loss_prot_logic_->UpdatePacketsPerFrame(min_packets_per_frame, |
| 321 | clock_->TimeInMilliseconds()); |
| 322 | } else { |
| 323 | loss_prot_logic_->UpdatePacketsPerFrameKey( |
| 324 | min_packets_per_frame, clock_->TimeInMilliseconds()); |
| 325 | } |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 326 | } |
| 327 | if (!delta_frame && encoded_length > 0) { |
| 328 | loss_prot_logic_->UpdateKeyFrameSize(static_cast<float>(encoded_length)); |
| 329 | } |
| 330 | |
| 331 | // Updating counters. |
| 332 | if (delta_frame) { |
| 333 | delta_frame_cnt_++; |
| 334 | } else { |
| 335 | key_frame_cnt_++; |
| 336 | } |
| 337 | } |
| 338 | |
| 339 | return VCM_OK; |
| 340 | } |
| 341 | |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 342 | void MediaOptimization::EnableFrameDropper(bool enable) { |
wuchengli@chromium.org | ae7cfd7 | 2014-06-30 08:01:47 +0000 | [diff] [blame] | 343 | CriticalSectionScoped lock(crit_sect_.get()); |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 344 | frame_dropper_->Enable(enable); |
| 345 | } |
| 346 | |
wuchengli@chromium.org | ae7cfd7 | 2014-06-30 08:01:47 +0000 | [diff] [blame] | 347 | void MediaOptimization::SuspendBelowMinBitrate(int threshold_bps, |
| 348 | int window_bps) { |
| 349 | CriticalSectionScoped lock(crit_sect_.get()); |
| 350 | assert(threshold_bps > 0 && window_bps >= 0); |
| 351 | suspension_threshold_bps_ = threshold_bps; |
| 352 | suspension_window_bps_ = window_bps; |
| 353 | suspension_enabled_ = true; |
| 354 | video_suspended_ = false; |
| 355 | } |
| 356 | |
| 357 | bool MediaOptimization::IsVideoSuspended() const { |
| 358 | CriticalSectionScoped lock(crit_sect_.get()); |
| 359 | return video_suspended_; |
| 360 | } |
| 361 | |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 362 | bool MediaOptimization::DropFrame() { |
wuchengli@chromium.org | ae7cfd7 | 2014-06-30 08:01:47 +0000 | [diff] [blame] | 363 | CriticalSectionScoped lock(crit_sect_.get()); |
andresp@webrtc.org | e682aa5 | 2013-12-19 10:59:48 +0000 | [diff] [blame] | 364 | UpdateIncomingFrameRate(); |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 365 | // Leak appropriate number of bytes. |
wuchengli@chromium.org | ae7cfd7 | 2014-06-30 08:01:47 +0000 | [diff] [blame] | 366 | frame_dropper_->Leak((uint32_t)(InputFrameRateInternal() + 0.5f)); |
henrik.lundin@webrtc.org | ce8e093 | 2013-11-18 12:18:43 +0000 | [diff] [blame] | 367 | if (video_suspended_) { |
henrik.lundin@webrtc.org | 544b17c | 2013-09-26 12:05:15 +0000 | [diff] [blame] | 368 | return true; // Drop all frames when muted. |
| 369 | } |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 370 | return frame_dropper_->DropFrame(); |
| 371 | } |
| 372 | |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 373 | void MediaOptimization::UpdateIncomingFrameRate() { |
| 374 | int64_t now = clock_->TimeInMilliseconds(); |
| 375 | if (incoming_frame_times_[0] == 0) { |
| 376 | // No shifting if this is the first time. |
| 377 | } else { |
| 378 | // Shift all times one step. |
| 379 | for (int32_t i = (kFrameCountHistorySize - 2); i >= 0; i--) { |
| 380 | incoming_frame_times_[i + 1] = incoming_frame_times_[i]; |
| 381 | } |
| 382 | } |
| 383 | incoming_frame_times_[0] = now; |
| 384 | ProcessIncomingFrameRate(now); |
| 385 | } |
| 386 | |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 387 | void MediaOptimization::PurgeOldFrameSamples(int64_t now_ms) { |
| 388 | while (!encoded_frame_samples_.empty()) { |
| 389 | if (now_ms - encoded_frame_samples_.front().time_complete_ms > |
stefan@webrtc.org | f4944d4 | 2013-03-18 17:04:52 +0000 | [diff] [blame] | 390 | kBitrateAverageWinMs) { |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 391 | encoded_frame_samples_.pop_front(); |
stefan@webrtc.org | f4944d4 | 2013-03-18 17:04:52 +0000 | [diff] [blame] | 392 | } else { |
| 393 | break; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 394 | } |
stefan@webrtc.org | f4944d4 | 2013-03-18 17:04:52 +0000 | [diff] [blame] | 395 | } |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 396 | } |
| 397 | |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 398 | void MediaOptimization::UpdateSentBitrate(int64_t now_ms) { |
| 399 | if (encoded_frame_samples_.empty()) { |
| 400 | avg_sent_bit_rate_bps_ = 0; |
stefan@webrtc.org | f4944d4 | 2013-03-18 17:04:52 +0000 | [diff] [blame] | 401 | return; |
| 402 | } |
pkasting@chromium.org | 4591fbd | 2014-11-20 22:28:14 +0000 | [diff] [blame] | 403 | size_t framesize_sum = 0; |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 404 | for (FrameSampleList::iterator it = encoded_frame_samples_.begin(); |
philipel | 9d3ab61 | 2015-12-21 04:12:39 -0800 | [diff] [blame] | 405 | it != encoded_frame_samples_.end(); ++it) { |
stefan@webrtc.org | f4944d4 | 2013-03-18 17:04:52 +0000 | [diff] [blame] | 406 | framesize_sum += it->size_bytes; |
| 407 | } |
| 408 | float denom = static_cast<float>( |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 409 | now_ms - encoded_frame_samples_.front().time_complete_ms); |
stefan@webrtc.org | f4944d4 | 2013-03-18 17:04:52 +0000 | [diff] [blame] | 410 | if (denom >= 1.0f) { |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 411 | avg_sent_bit_rate_bps_ = |
stefan@webrtc.org | 0bae1fa | 2014-11-05 14:05:29 +0000 | [diff] [blame] | 412 | static_cast<uint32_t>(framesize_sum * 8.0f * 1000.0f / denom + 0.5f); |
stefan@webrtc.org | f4944d4 | 2013-03-18 17:04:52 +0000 | [diff] [blame] | 413 | } else { |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 414 | avg_sent_bit_rate_bps_ = framesize_sum * 8; |
stefan@webrtc.org | f4944d4 | 2013-03-18 17:04:52 +0000 | [diff] [blame] | 415 | } |
| 416 | } |
| 417 | |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 418 | void MediaOptimization::UpdateSentFramerate() { |
| 419 | if (encoded_frame_samples_.size() <= 1) { |
| 420 | avg_sent_framerate_ = encoded_frame_samples_.size(); |
stefan@webrtc.org | f4944d4 | 2013-03-18 17:04:52 +0000 | [diff] [blame] | 421 | return; |
| 422 | } |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 423 | int denom = encoded_frame_samples_.back().timestamp - |
| 424 | encoded_frame_samples_.front().timestamp; |
stefan@webrtc.org | f4944d4 | 2013-03-18 17:04:52 +0000 | [diff] [blame] | 425 | if (denom > 0) { |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 426 | avg_sent_framerate_ = |
| 427 | (90000 * (encoded_frame_samples_.size() - 1) + denom / 2) / denom; |
stefan@webrtc.org | f4944d4 | 2013-03-18 17:04:52 +0000 | [diff] [blame] | 428 | } else { |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 429 | avg_sent_framerate_ = encoded_frame_samples_.size(); |
stefan@webrtc.org | f4944d4 | 2013-03-18 17:04:52 +0000 | [diff] [blame] | 430 | } |
| 431 | } |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 432 | |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 433 | // Allowing VCM to keep track of incoming frame rate. |
| 434 | void MediaOptimization::ProcessIncomingFrameRate(int64_t now) { |
| 435 | int32_t num = 0; |
| 436 | int32_t nr_of_frames = 0; |
| 437 | for (num = 1; num < (kFrameCountHistorySize - 1); ++num) { |
| 438 | if (incoming_frame_times_[num] <= 0 || |
| 439 | // don't use data older than 2 s |
| 440 | now - incoming_frame_times_[num] > kFrameHistoryWinMs) { |
| 441 | break; |
| 442 | } else { |
| 443 | nr_of_frames++; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 444 | } |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 445 | } |
| 446 | if (num > 1) { |
Erik Språng | 66a641a | 2015-06-11 14:20:07 +0200 | [diff] [blame] | 447 | const int64_t diff = |
| 448 | incoming_frame_times_[0] - incoming_frame_times_[num - 1]; |
| 449 | incoming_frame_rate_ = 0.0; // No frame rate estimate available. |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 450 | if (diff > 0) { |
| 451 | incoming_frame_rate_ = nr_of_frames * 1000.0f / static_cast<float>(diff); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 452 | } |
henrik.lundin@webrtc.org | bec11ef | 2013-09-23 19:54:25 +0000 | [diff] [blame] | 453 | } |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 454 | } |
| 455 | |
henrik.lundin@webrtc.org | ce8e093 | 2013-11-18 12:18:43 +0000 | [diff] [blame] | 456 | void MediaOptimization::CheckSuspendConditions() { |
pbos | 7367463 | 2015-10-29 15:45:00 -0700 | [diff] [blame] | 457 | // Check conditions for SuspendBelowMinBitrate. |video_target_bitrate_| is in |
| 458 | // bps. |
henrik.lundin@webrtc.org | ce8e093 | 2013-11-18 12:18:43 +0000 | [diff] [blame] | 459 | if (suspension_enabled_) { |
| 460 | if (!video_suspended_) { |
henrik.lundin@webrtc.org | 544b17c | 2013-09-26 12:05:15 +0000 | [diff] [blame] | 461 | // Check if we just went below the threshold. |
pbos | 7367463 | 2015-10-29 15:45:00 -0700 | [diff] [blame] | 462 | if (video_target_bitrate_ < suspension_threshold_bps_) { |
henrik.lundin@webrtc.org | ce8e093 | 2013-11-18 12:18:43 +0000 | [diff] [blame] | 463 | video_suspended_ = true; |
henrik.lundin@webrtc.org | 544b17c | 2013-09-26 12:05:15 +0000 | [diff] [blame] | 464 | } |
| 465 | } else { |
henrik.lundin@webrtc.org | ce8e093 | 2013-11-18 12:18:43 +0000 | [diff] [blame] | 466 | // Video is already suspended. Check if we just went over the threshold |
henrik.lundin@webrtc.org | 544b17c | 2013-09-26 12:05:15 +0000 | [diff] [blame] | 467 | // with a margin. |
pbos | 7367463 | 2015-10-29 15:45:00 -0700 | [diff] [blame] | 468 | if (video_target_bitrate_ > |
henrik.lundin@webrtc.org | ce8e093 | 2013-11-18 12:18:43 +0000 | [diff] [blame] | 469 | suspension_threshold_bps_ + suspension_window_bps_) { |
| 470 | video_suspended_ = false; |
henrik.lundin@webrtc.org | 544b17c | 2013-09-26 12:05:15 +0000 | [diff] [blame] | 471 | } |
| 472 | } |
| 473 | } |
| 474 | } |
| 475 | |
stefan@webrtc.org | a64300a | 2013-03-04 15:24:40 +0000 | [diff] [blame] | 476 | } // namespace media_optimization |
| 477 | } // namespace webrtc |