blob: 4afe47dd8d0853ec95f4aca3debdacc90e37e8d8 [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:25 +00001/*
stefan@webrtc.orgc35f5ce2012-04-11 07:42:25 +00002 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
niklase@google.com470e71d2011-07-07 08:21:25 +00003 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "modules/video_coding/media_opt_util.h"
niklase@google.com470e71d2011-07-07 08:21:25 +000012
niklase@google.com470e71d2011-07-07 08:21:25 +000013#include <float.h>
14#include <limits.h>
pbos@webrtc.orga4407322013-07-16 12:32:05 +000015#include <math.h>
niklase@google.com470e71d2011-07-07 08:21:25 +000016
philipel9d3ab612015-12-21 04:12:39 -080017#include <algorithm>
brandtr05f845d2016-11-16 22:59:39 -080018#include <limits>
philipel9d3ab612015-12-21 04:12:39 -080019
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020020#include "modules/include/module_common_types.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020021#include "modules/video_coding/fec_rate_table.h"
Yves Gerey665174f2018-06-19 15:03:05 +020022#include "modules/video_coding/include/video_coding_defines.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "modules/video_coding/nack_fec_tables.h"
Sergio Garcia Murillo43800f92018-06-21 16:16:38 +020024#include "modules/video_coding/utility/simulcast_rate_allocator.h"
mikhal@webrtc.org0e7d9d82011-12-19 19:04:49 +000025
niklase@google.com470e71d2011-07-07 08:21:25 +000026namespace webrtc {
Peter Boström9cb1f302015-04-01 11:39:49 +020027// Max value of loss rates in off-line model
28static const int kPacketLossMax = 129;
29
stefan@webrtc.orga64300a2013-03-04 15:24:40 +000030namespace media_optimization {
niklase@google.com470e71d2011-07-07 08:21:25 +000031
Stefan Holmerdbdb3a02018-07-17 16:03:46 +020032VCMProtectionParameters::VCMProtectionParameters()
33 : rtt(0),
34 lossPr(0.0f),
35 bitRate(0.0f),
36 packetsPerFrame(0.0f),
37 packetsPerFrameKey(0.0f),
38 frameRate(0.0f),
39 keyFrameSize(0.0f),
40 fecRateDelta(0),
41 fecRateKey(0),
42 codecWidth(0),
43 codecHeight(0),
44 numLayers(1) {}
45
Peter Boström9cb1f302015-04-01 11:39:49 +020046VCMProtectionMethod::VCMProtectionMethod()
47 : _effectivePacketLoss(0),
48 _protectionFactorK(0),
49 _protectionFactorD(0),
50 _scaleProtKey(2.0f),
51 _maxPayloadSize(1460),
Peter Boström9cb1f302015-04-01 11:39:49 +020052 _corrFecCost(1.0),
philipel9d3ab612015-12-21 04:12:39 -080053 _type(kNone) {}
mikhal@webrtc.orga057a952011-08-26 21:17:34 +000054
pbosc0430522016-05-01 17:19:05 -070055VCMProtectionMethod::~VCMProtectionMethod() {}
marpan@google.com86548c62011-07-12 17:12:57 +000056
Stefan Holmerdbdb3a02018-07-17 16:03:46 +020057enum VCMProtectionMethodEnum VCMProtectionMethod::Type() const {
58 return _type;
59}
60
61uint8_t VCMProtectionMethod::RequiredPacketLossER() {
62 return _effectivePacketLoss;
63}
64
65uint8_t VCMProtectionMethod::RequiredProtectionFactorK() {
66 return _protectionFactorK;
67}
68
69uint8_t VCMProtectionMethod::RequiredProtectionFactorD() {
70 return _protectionFactorD;
71}
72
73bool VCMProtectionMethod::RequiredUepProtectionK() {
74 return _useUepProtectionK;
75}
76
77bool VCMProtectionMethod::RequiredUepProtectionD() {
78 return _useUepProtectionD;
79}
80
81int VCMProtectionMethod::MaxFramesFec() const {
82 return 1;
83}
84
pkasting@chromium.org16825b12015-01-12 21:51:21 +000085VCMNackFecMethod::VCMNackFecMethod(int64_t lowRttNackThresholdMs,
86 int64_t highRttNackThresholdMs)
stefan@webrtc.org932ab182011-11-29 11:33:31 +000087 : VCMFecMethod(),
88 _lowRttNackMs(lowRttNackThresholdMs),
stefan@webrtc.orgc35f5ce2012-04-11 07:42:25 +000089 _highRttNackMs(highRttNackThresholdMs),
90 _maxFramesFec(1) {
stefan@webrtc.org932ab182011-11-29 11:33:31 +000091 assert(lowRttNackThresholdMs >= -1 && highRttNackThresholdMs >= -1);
92 assert(highRttNackThresholdMs == -1 ||
93 lowRttNackThresholdMs <= highRttNackThresholdMs);
94 assert(lowRttNackThresholdMs > -1 || highRttNackThresholdMs == -1);
95 _type = kNackFec;
mikhal@google.com77408882011-07-22 22:05:25 +000096}
97
philipel9d3ab612015-12-21 04:12:39 -080098VCMNackFecMethod::~VCMNackFecMethod() {
99 //
mikhal@google.com77408882011-07-22 22:05:25 +0000100}
philipel9d3ab612015-12-21 04:12:39 -0800101bool VCMNackFecMethod::ProtectionFactor(
102 const VCMProtectionParameters* parameters) {
103 // Hybrid Nack FEC has three operational modes:
104 // 1. Low RTT (below kLowRttNackMs) - Nack only: Set FEC rate
105 // (_protectionFactorD) to zero. -1 means no FEC.
106 // 2. High RTT (above _highRttNackMs) - FEC Only: Keep FEC factors.
107 // -1 means always allow NACK.
108 // 3. Medium RTT values - Hybrid mode: We will only nack the
109 // residual following the decoding of the FEC (refer to JB logic). FEC
110 // delta protection factor will be adjusted based on the RTT.
mikhal@google.com022716b2011-07-20 23:12:57 +0000111
philipel9d3ab612015-12-21 04:12:39 -0800112 // Otherwise: we count on FEC; if the RTT is below a threshold, then we
113 // nack the residual, based on a decision made in the JB.
mikhal@google.com022716b2011-07-20 23:12:57 +0000114
philipel9d3ab612015-12-21 04:12:39 -0800115 // Compute the protection factors
116 VCMFecMethod::ProtectionFactor(parameters);
117 if (_lowRttNackMs == -1 || parameters->rtt < _lowRttNackMs) {
118 _protectionFactorD = 0;
119 VCMFecMethod::UpdateProtectionFactorD(_protectionFactorD);
mikhal@google.com320813c2011-08-03 20:47:50 +0000120
mikhal@google.com679450f2011-08-01 22:14:58 +0000121 // When in Hybrid mode (RTT range), adjust FEC rates based on the
122 // RTT (NACK effectiveness) - adjustment factor is in the range [0,1].
philipel9d3ab612015-12-21 04:12:39 -0800123 } else if (_highRttNackMs == -1 || parameters->rtt < _highRttNackMs) {
124 // TODO(mikhal): Disabling adjustment temporarily.
125 // uint16_t rttIndex = (uint16_t) parameters->rtt;
126 float adjustRtt = 1.0f; // (float)VCMNackFecTable[rttIndex] / 100.0f;
niklase@google.com470e71d2011-07-07 08:21:25 +0000127
philipel9d3ab612015-12-21 04:12:39 -0800128 // Adjust FEC with NACK on (for delta frame only)
129 // table depends on RTT relative to rttMax (NACK Threshold)
130 _protectionFactorD = static_cast<uint8_t>(
131 adjustRtt * static_cast<float>(_protectionFactorD));
132 // update FEC rates after applying adjustment
133 VCMFecMethod::UpdateProtectionFactorD(_protectionFactorD);
134 }
mikhal@google.com679450f2011-08-01 22:14:58 +0000135
philipel9d3ab612015-12-21 04:12:39 -0800136 return true;
mikhal@google.com022716b2011-07-20 23:12:57 +0000137}
niklase@google.com470e71d2011-07-07 08:21:25 +0000138
stefan@webrtc.orgc35f5ce2012-04-11 07:42:25 +0000139int VCMNackFecMethod::ComputeMaxFramesFec(
140 const VCMProtectionParameters* parameters) {
141 if (parameters->numLayers > 2) {
142 // For more than 2 temporal layers we will only have FEC on the base layer,
143 // and the base layers will be pretty far apart. Therefore we force one
144 // frame FEC.
145 return 1;
146 }
147 // We set the max number of frames to base the FEC on so that on average
148 // we will have complete frames in one RTT. Note that this is an upper
149 // bound, and that the actual number of frames used for FEC is decided by the
150 // RTP module based on the actual number of packets and the protection factor.
philipel9d3ab612015-12-21 04:12:39 -0800151 float base_layer_framerate =
152 parameters->frameRate /
stefan@webrtc.orgc35f5ce2012-04-11 07:42:25 +0000153 static_cast<float>(1 << (parameters->numLayers - 1));
philipel9d3ab612015-12-21 04:12:39 -0800154 int max_frames_fec = std::max(
155 static_cast<int>(2.0f * base_layer_framerate * parameters->rtt / 1000.0f +
156 0.5f),
157 1);
stefan@webrtc.orgc35f5ce2012-04-11 07:42:25 +0000158 // |kUpperLimitFramesFec| is the upper limit on how many frames we
159 // allow any FEC to be based on.
160 if (max_frames_fec > kUpperLimitFramesFec) {
161 max_frames_fec = kUpperLimitFramesFec;
162 }
163 return max_frames_fec;
164}
165
166int VCMNackFecMethod::MaxFramesFec() const {
167 return _maxFramesFec;
168}
169
marpan@webrtc.org88ad06b2012-04-20 16:05:24 +0000170bool VCMNackFecMethod::BitRateTooLowForFec(
171 const VCMProtectionParameters* parameters) {
172 // Bitrate below which we turn off FEC, regardless of reported packet loss.
173 // The condition should depend on resolution and content. For now, use
174 // threshold on bytes per frame, with some effect for the frame size.
175 // The condition for turning off FEC is also based on other factors,
176 // such as |_numLayers|, |_maxFramesFec|, and |_rtt|.
177 int estimate_bytes_per_frame = 1000 * BitsPerFrame(parameters) / 8;
178 int max_bytes_per_frame = kMaxBytesPerFrameForFec;
179 int num_pixels = parameters->codecWidth * parameters->codecHeight;
180 if (num_pixels <= 352 * 288) {
181 max_bytes_per_frame = kMaxBytesPerFrameForFecLow;
182 } else if (num_pixels > 640 * 480) {
183 max_bytes_per_frame = kMaxBytesPerFrameForFecHigh;
184 }
philipel9d3ab612015-12-21 04:12:39 -0800185 // TODO(marpan): add condition based on maximum frames used for FEC,
marpan@webrtc.org88ad06b2012-04-20 16:05:24 +0000186 // and expand condition based on frame size.
pkasting@chromium.org16825b12015-01-12 21:51:21 +0000187 // Max round trip time threshold in ms.
188 const int64_t kMaxRttTurnOffFec = 200;
marpan@webrtc.org88ad06b2012-04-20 16:05:24 +0000189 if (estimate_bytes_per_frame < max_bytes_per_frame &&
philipel9d3ab612015-12-21 04:12:39 -0800190 parameters->numLayers < 3 && parameters->rtt < kMaxRttTurnOffFec) {
marpan@webrtc.org88ad06b2012-04-20 16:05:24 +0000191 return true;
192 }
193 return false;
194}
195
philipel9d3ab612015-12-21 04:12:39 -0800196bool VCMNackFecMethod::EffectivePacketLoss(
197 const VCMProtectionParameters* parameters) {
198 // Set the effective packet loss for encoder (based on FEC code).
199 // Compute the effective packet loss and residual packet loss due to FEC.
200 VCMFecMethod::EffectivePacketLoss(parameters);
201 return true;
mikhal@google.com022716b2011-07-20 23:12:57 +0000202}
niklase@google.com470e71d2011-07-07 08:21:25 +0000203
philipel9d3ab612015-12-21 04:12:39 -0800204bool VCMNackFecMethod::UpdateParameters(
205 const VCMProtectionParameters* parameters) {
206 ProtectionFactor(parameters);
207 EffectivePacketLoss(parameters);
208 _maxFramesFec = ComputeMaxFramesFec(parameters);
209 if (BitRateTooLowForFec(parameters)) {
210 _protectionFactorK = 0;
211 _protectionFactorD = 0;
212 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000213
philipel9d3ab612015-12-21 04:12:39 -0800214 // Protection/fec rates obtained above are defined relative to total number
215 // of packets (total rate: source + fec) FEC in RTP module assumes
216 // protection factor is defined relative to source number of packets so we
217 // should convert the factor to reduce mismatch between mediaOpt's rate and
218 // the actual one
219 _protectionFactorK = VCMFecMethod::ConvertFECRate(_protectionFactorK);
220 _protectionFactorD = VCMFecMethod::ConvertFECRate(_protectionFactorD);
niklase@google.com470e71d2011-07-07 08:21:25 +0000221
philipel9d3ab612015-12-21 04:12:39 -0800222 return true;
niklase@google.com470e71d2011-07-07 08:21:25 +0000223}
224
philipel9d3ab612015-12-21 04:12:39 -0800225VCMNackMethod::VCMNackMethod() : VCMProtectionMethod() {
226 _type = kNack;
mikhal@webrtc.orga057a952011-08-26 21:17:34 +0000227}
228
philipel9d3ab612015-12-21 04:12:39 -0800229VCMNackMethod::~VCMNackMethod() {
230 //
mikhal@webrtc.orga057a952011-08-26 21:17:34 +0000231}
232
philipel9d3ab612015-12-21 04:12:39 -0800233bool VCMNackMethod::EffectivePacketLoss(
234 const VCMProtectionParameters* parameter) {
235 // Effective Packet Loss, NA in current version.
236 _effectivePacketLoss = 0;
237 return true;
niklase@google.com470e71d2011-07-07 08:21:25 +0000238}
239
philipel9d3ab612015-12-21 04:12:39 -0800240bool VCMNackMethod::UpdateParameters(
241 const VCMProtectionParameters* parameters) {
242 // Compute the effective packet loss
243 EffectivePacketLoss(parameters);
niklase@google.com470e71d2011-07-07 08:21:25 +0000244
philipel9d3ab612015-12-21 04:12:39 -0800245 // nackCost = (bitRate - nackCost) * (lossPr)
246 return true;
niklase@google.com470e71d2011-07-07 08:21:25 +0000247}
248
philipel9d3ab612015-12-21 04:12:39 -0800249VCMFecMethod::VCMFecMethod() : VCMProtectionMethod() {
250 _type = kFec;
mikhal@webrtc.orga057a952011-08-26 21:17:34 +0000251}
philipel9d3ab612015-12-21 04:12:39 -0800252VCMFecMethod::~VCMFecMethod() {
253 //
mikhal@webrtc.orga057a952011-08-26 21:17:34 +0000254}
255
philipel9d3ab612015-12-21 04:12:39 -0800256uint8_t VCMFecMethod::BoostCodeRateKey(uint8_t packetFrameDelta,
257 uint8_t packetFrameKey) const {
258 uint8_t boostRateKey = 2;
259 // Default: ratio scales the FEC protection up for I frames
260 uint8_t ratio = 1;
niklase@google.com470e71d2011-07-07 08:21:25 +0000261
philipel9d3ab612015-12-21 04:12:39 -0800262 if (packetFrameDelta > 0) {
263 ratio = (int8_t)(packetFrameKey / packetFrameDelta);
264 }
265 ratio = VCM_MAX(boostRateKey, ratio);
niklase@google.com470e71d2011-07-07 08:21:25 +0000266
philipel9d3ab612015-12-21 04:12:39 -0800267 return ratio;
niklase@google.com470e71d2011-07-07 08:21:25 +0000268}
269
philipel9d3ab612015-12-21 04:12:39 -0800270uint8_t VCMFecMethod::ConvertFECRate(uint8_t codeRateRTP) const {
271 return static_cast<uint8_t>(VCM_MIN(
272 255,
273 (0.5 + 255.0 * codeRateRTP / static_cast<float>(255 - codeRateRTP))));
niklase@google.com470e71d2011-07-07 08:21:25 +0000274}
275
mikhal@google.com679450f2011-08-01 22:14:58 +0000276// Update FEC with protectionFactorD
philipel9d3ab612015-12-21 04:12:39 -0800277void VCMFecMethod::UpdateProtectionFactorD(uint8_t protectionFactorD) {
278 _protectionFactorD = protectionFactorD;
mikhal@google.com679450f2011-08-01 22:14:58 +0000279}
280
mikhal@webrtc.orgd0752c32011-10-19 15:48:30 +0000281// Update FEC with protectionFactorK
philipel9d3ab612015-12-21 04:12:39 -0800282void VCMFecMethod::UpdateProtectionFactorK(uint8_t protectionFactorK) {
283 _protectionFactorK = protectionFactorK;
mikhal@webrtc.orgd0752c32011-10-19 15:48:30 +0000284}
285
philipel9d3ab612015-12-21 04:12:39 -0800286bool VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters) {
287 // FEC PROTECTION SETTINGS: varies with packet loss and bitrate
niklase@google.com470e71d2011-07-07 08:21:25 +0000288
philipel9d3ab612015-12-21 04:12:39 -0800289 // No protection if (filtered) packetLoss is 0
brandtr05f845d2016-11-16 22:59:39 -0800290 uint8_t packetLoss = static_cast<uint8_t>(255 * parameters->lossPr);
philipel9d3ab612015-12-21 04:12:39 -0800291 if (packetLoss == 0) {
292 _protectionFactorK = 0;
293 _protectionFactorD = 0;
niklase@google.com470e71d2011-07-07 08:21:25 +0000294 return true;
philipel9d3ab612015-12-21 04:12:39 -0800295 }
296
297 // Parameters for FEC setting:
298 // first partition size, thresholds, table pars, spatial resoln fac.
299
300 // First partition protection: ~ 20%
brandtr05f845d2016-11-16 22:59:39 -0800301 uint8_t firstPartitionProt = static_cast<uint8_t>(255 * 0.20);
philipel9d3ab612015-12-21 04:12:39 -0800302
303 // Minimum protection level needed to generate one FEC packet for one
304 // source packet/frame (in RTP sender)
305 uint8_t minProtLevelFec = 85;
306
307 // Threshold on packetLoss and bitRrate/frameRate (=average #packets),
308 // above which we allocate protection to cover at least first partition.
309 uint8_t lossThr = 0;
310 uint8_t packetNumThr = 1;
311
312 // Parameters for range of rate index of table.
313 const uint8_t ratePar1 = 5;
314 const uint8_t ratePar2 = 49;
315
316 // Spatial resolution size, relative to a reference size.
317 float spatialSizeToRef =
318 static_cast<float>(parameters->codecWidth * parameters->codecHeight) /
319 (static_cast<float>(704 * 576));
320 // resolnFac: This parameter will generally increase/decrease the FEC rate
321 // (for fixed bitRate and packetLoss) based on system size.
322 // Use a smaller exponent (< 1) to control/soften system size effect.
323 const float resolnFac = 1.0 / powf(spatialSizeToRef, 0.3f);
324
325 const int bitRatePerFrame = BitsPerFrame(parameters);
326
327 // Average number of packets per frame (source and fec):
brandtr05f845d2016-11-16 22:59:39 -0800328 const uint8_t avgTotPackets = static_cast<uint8_t>(
329 std::min(static_cast<float>(std::numeric_limits<uint8_t>::max()),
Yves Gerey665174f2018-06-19 15:03:05 +0200330 1.5f + static_cast<float>(bitRatePerFrame) * 1000.0f /
331 static_cast<float>(8.0 * _maxPayloadSize)));
philipel9d3ab612015-12-21 04:12:39 -0800332
333 // FEC rate parameters: for P and I frame
334 uint8_t codeRateDelta = 0;
335 uint8_t codeRateKey = 0;
336
337 // Get index for table: the FEC protection depends on an effective rate.
338 // The range on the rate index corresponds to rates (bps)
339 // from ~200k to ~8000k, for 30fps
340 const uint16_t effRateFecTable =
341 static_cast<uint16_t>(resolnFac * bitRatePerFrame);
brandtr05f845d2016-11-16 22:59:39 -0800342 uint8_t rateIndexTable = static_cast<uint8_t>(
343 VCM_MAX(VCM_MIN((effRateFecTable - ratePar1) / ratePar1, ratePar2), 0));
philipel9d3ab612015-12-21 04:12:39 -0800344
345 // Restrict packet loss range to 50:
346 // current tables defined only up to 50%
347 if (packetLoss >= kPacketLossMax) {
348 packetLoss = kPacketLossMax - 1;
349 }
350 uint16_t indexTable = rateIndexTable * kPacketLossMax + packetLoss;
351
352 // Check on table index
brandtr71b1c1f2017-01-12 06:16:24 -0800353 RTC_DCHECK_LT(indexTable, kFecRateTableSize);
philipel9d3ab612015-12-21 04:12:39 -0800354
355 // Protection factor for P frame
brandtr71b1c1f2017-01-12 06:16:24 -0800356 codeRateDelta = kFecRateTable[indexTable];
philipel9d3ab612015-12-21 04:12:39 -0800357
358 if (packetLoss > lossThr && avgTotPackets > packetNumThr) {
359 // Set a minimum based on first partition size.
360 if (codeRateDelta < firstPartitionProt) {
361 codeRateDelta = firstPartitionProt;
362 }
363 }
364
365 // Check limit on amount of protection for P frame; 50% is max.
366 if (codeRateDelta >= kPacketLossMax) {
367 codeRateDelta = kPacketLossMax - 1;
368 }
369
philipel9d3ab612015-12-21 04:12:39 -0800370 // For Key frame:
371 // Effectively at a higher rate, so we scale/boost the rate
372 // The boost factor may depend on several factors: ratio of packet
373 // number of I to P frames, how much protection placed on P frames, etc.
brandtr05f845d2016-11-16 22:59:39 -0800374 const uint8_t packetFrameDelta =
375 static_cast<uint8_t>(0.5 + parameters->packetsPerFrame);
philipel9d3ab612015-12-21 04:12:39 -0800376 const uint8_t packetFrameKey =
brandtr05f845d2016-11-16 22:59:39 -0800377 static_cast<uint8_t>(0.5 + parameters->packetsPerFrameKey);
philipel9d3ab612015-12-21 04:12:39 -0800378 const uint8_t boostKey = BoostCodeRateKey(packetFrameDelta, packetFrameKey);
379
brandtr05f845d2016-11-16 22:59:39 -0800380 rateIndexTable = static_cast<uint8_t>(VCM_MAX(
philipel9d3ab612015-12-21 04:12:39 -0800381 VCM_MIN(1 + (boostKey * effRateFecTable - ratePar1) / ratePar1, ratePar2),
brandtr05f845d2016-11-16 22:59:39 -0800382 0));
philipel9d3ab612015-12-21 04:12:39 -0800383 uint16_t indexTableKey = rateIndexTable * kPacketLossMax + packetLoss;
384
brandtr71b1c1f2017-01-12 06:16:24 -0800385 indexTableKey = VCM_MIN(indexTableKey, kFecRateTableSize);
philipel9d3ab612015-12-21 04:12:39 -0800386
387 // Check on table index
brandtr71b1c1f2017-01-12 06:16:24 -0800388 assert(indexTableKey < kFecRateTableSize);
philipel9d3ab612015-12-21 04:12:39 -0800389
390 // Protection factor for I frame
brandtr71b1c1f2017-01-12 06:16:24 -0800391 codeRateKey = kFecRateTable[indexTableKey];
philipel9d3ab612015-12-21 04:12:39 -0800392
393 // Boosting for Key frame.
394 int boostKeyProt = _scaleProtKey * codeRateDelta;
395 if (boostKeyProt >= kPacketLossMax) {
396 boostKeyProt = kPacketLossMax - 1;
397 }
398
399 // Make sure I frame protection is at least larger than P frame protection,
400 // and at least as high as filtered packet loss.
401 codeRateKey = static_cast<uint8_t>(
402 VCM_MAX(packetLoss, VCM_MAX(boostKeyProt, codeRateKey)));
403
404 // Check limit on amount of protection for I frame: 50% is max.
405 if (codeRateKey >= kPacketLossMax) {
406 codeRateKey = kPacketLossMax - 1;
407 }
408
409 _protectionFactorK = codeRateKey;
410 _protectionFactorD = codeRateDelta;
411
412 // Generally there is a rate mis-match between the FEC cost estimated
413 // in mediaOpt and the actual FEC cost sent out in RTP module.
414 // This is more significant at low rates (small # of source packets), where
415 // the granularity of the FEC decreases. In this case, non-zero protection
416 // in mediaOpt may generate 0 FEC packets in RTP sender (since actual #FEC
417 // is based on rounding off protectionFactor on actual source packet number).
418 // The correction factor (_corrFecCost) attempts to corrects this, at least
419 // for cases of low rates (small #packets) and low protection levels.
420
421 float numPacketsFl = 1.0f + (static_cast<float>(bitRatePerFrame) * 1000.0 /
422 static_cast<float>(8.0 * _maxPayloadSize) +
423 0.5);
424
425 const float estNumFecGen =
426 0.5f + static_cast<float>(_protectionFactorD * numPacketsFl / 255.0f);
427
428 // We reduce cost factor (which will reduce overhead for FEC and
429 // hybrid method) and not the protectionFactor.
430 _corrFecCost = 1.0f;
431 if (estNumFecGen < 1.1f && _protectionFactorD < minProtLevelFec) {
432 _corrFecCost = 0.5f;
433 }
434 if (estNumFecGen < 0.9f && _protectionFactorD < minProtLevelFec) {
435 _corrFecCost = 0.0f;
436 }
437
philipel9d3ab612015-12-21 04:12:39 -0800438 // DONE WITH FEC PROTECTION SETTINGS
439 return true;
niklase@google.com470e71d2011-07-07 08:21:25 +0000440}
441
mikhal@webrtc.org0e7d9d82011-12-19 19:04:49 +0000442int VCMFecMethod::BitsPerFrame(const VCMProtectionParameters* parameters) {
443 // When temporal layers are available FEC will only be applied on the base
444 // layer.
Erik Språngd92288f2018-07-04 10:07:40 +0200445 const float bitRateRatio =
446 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
447 parameters->numLayers, 0);
thakis@chromium.org65bc2542012-08-13 19:26:12 +0000448 float frameRateRatio = powf(1 / 2.0, parameters->numLayers - 1);
mikhal@webrtc.org0e7d9d82011-12-19 19:04:49 +0000449 float bitRate = parameters->bitRate * bitRateRatio;
450 float frameRate = parameters->frameRate * frameRateRatio;
451
452 // TODO(mikhal): Update factor following testing.
453 float adjustmentFactor = 1;
454
Peter Boström5e8351b2016-01-28 23:55:29 +0100455 if (frameRate < 1.0f)
456 frameRate = 1.0f;
mikhal@webrtc.org0e7d9d82011-12-19 19:04:49 +0000457 // Average bits per frame (units of kbits)
458 return static_cast<int>(adjustmentFactor * bitRate / frameRate);
459}
460
philipel9d3ab612015-12-21 04:12:39 -0800461bool VCMFecMethod::EffectivePacketLoss(
462 const VCMProtectionParameters* parameters) {
463 // Effective packet loss to encoder is based on RPL (residual packet loss)
464 // this is a soft setting based on degree of FEC protection
465 // RPL = received/input packet loss - average_FEC_recovery
466 // note: received/input packet loss may be filtered based on FilteredLoss
niklase@google.com470e71d2011-07-07 08:21:25 +0000467
philipel9d3ab612015-12-21 04:12:39 -0800468 // Effective Packet Loss, NA in current version.
469 _effectivePacketLoss = 0;
niklase@google.com470e71d2011-07-07 08:21:25 +0000470
philipel9d3ab612015-12-21 04:12:39 -0800471 return true;
niklase@google.com470e71d2011-07-07 08:21:25 +0000472}
473
philipel9d3ab612015-12-21 04:12:39 -0800474bool VCMFecMethod::UpdateParameters(const VCMProtectionParameters* parameters) {
475 // Compute the protection factor
476 ProtectionFactor(parameters);
niklase@google.com470e71d2011-07-07 08:21:25 +0000477
philipel9d3ab612015-12-21 04:12:39 -0800478 // Compute the effective packet loss
479 EffectivePacketLoss(parameters);
niklase@google.com470e71d2011-07-07 08:21:25 +0000480
philipel9d3ab612015-12-21 04:12:39 -0800481 // Protection/fec rates obtained above is defined relative to total number
482 // of packets (total rate: source+fec) FEC in RTP module assumes protection
483 // factor is defined relative to source number of packets so we should
484 // convert the factor to reduce mismatch between mediaOpt suggested rate and
485 // the actual rate
486 _protectionFactorK = ConvertFECRate(_protectionFactorK);
487 _protectionFactorD = ConvertFECRate(_protectionFactorD);
niklase@google.com470e71d2011-07-07 08:21:25 +0000488
philipel9d3ab612015-12-21 04:12:39 -0800489 return true;
niklase@google.com470e71d2011-07-07 08:21:25 +0000490}
philipel9d3ab612015-12-21 04:12:39 -0800491VCMLossProtectionLogic::VCMLossProtectionLogic(int64_t nowMs)
492 : _currentParameters(),
493 _rtt(0),
494 _lossPr(0.0f),
495 _bitRate(0.0f),
496 _frameRate(0.0f),
497 _keyFrameSize(0.0f),
498 _fecRateKey(0),
499 _fecRateDelta(0),
500 _lastPrUpdateT(0),
501 _lossPr255(0.9999f),
502 _lossPrHistory(),
503 _shortMaxLossPr255(0),
504 _packetsPerFrame(0.9999f),
505 _packetsPerFrameKey(0.9999f),
506 _codecWidth(0),
507 _codecHeight(0),
508 _numLayers(1) {
509 Reset(nowMs);
niklase@google.com470e71d2011-07-07 08:21:25 +0000510}
511
philipel9d3ab612015-12-21 04:12:39 -0800512VCMLossProtectionLogic::~VCMLossProtectionLogic() {
513 Release();
niklase@google.com470e71d2011-07-07 08:21:25 +0000514}
515
pbos@webrtc.orgcade82c2015-03-12 10:39:24 +0000516void VCMLossProtectionLogic::SetMethod(
517 enum VCMProtectionMethodEnum newMethodType) {
pbosba8c15b2015-07-14 09:36:34 -0700518 if (_selectedMethod && _selectedMethod->Type() == newMethodType)
519 return;
mikhal@webrtc.orga057a952011-08-26 21:17:34 +0000520
philipel9d3ab612015-12-21 04:12:39 -0800521 switch (newMethodType) {
pbos@webrtc.orgcade82c2015-03-12 10:39:24 +0000522 case kNack:
pbosba8c15b2015-07-14 09:36:34 -0700523 _selectedMethod.reset(new VCMNackMethod());
pbos@webrtc.orgcade82c2015-03-12 10:39:24 +0000524 break;
525 case kFec:
pbosba8c15b2015-07-14 09:36:34 -0700526 _selectedMethod.reset(new VCMFecMethod());
pbos@webrtc.orgcade82c2015-03-12 10:39:24 +0000527 break;
528 case kNackFec:
pbosba8c15b2015-07-14 09:36:34 -0700529 _selectedMethod.reset(new VCMNackFecMethod(kLowRttNackMs, -1));
pbos@webrtc.orgcade82c2015-03-12 10:39:24 +0000530 break;
531 case kNone:
pbosba8c15b2015-07-14 09:36:34 -0700532 _selectedMethod.reset();
pbos@webrtc.orgcade82c2015-03-12 10:39:24 +0000533 break;
534 }
535 UpdateMethod();
niklase@google.com470e71d2011-07-07 08:21:25 +0000536}
537
philipel9d3ab612015-12-21 04:12:39 -0800538void VCMLossProtectionLogic::UpdateRtt(int64_t rtt) {
539 _rtt = rtt;
niklase@google.com470e71d2011-07-07 08:21:25 +0000540}
541
philipel9d3ab612015-12-21 04:12:39 -0800542void VCMLossProtectionLogic::UpdateMaxLossHistory(uint8_t lossPr255,
543 int64_t now) {
544 if (_lossPrHistory[0].timeMs >= 0 &&
545 now - _lossPrHistory[0].timeMs < kLossPrShortFilterWinMs) {
546 if (lossPr255 > _shortMaxLossPr255) {
547 _shortMaxLossPr255 = lossPr255;
niklase@google.com470e71d2011-07-07 08:21:25 +0000548 }
philipel9d3ab612015-12-21 04:12:39 -0800549 } else {
550 // Only add a new value to the history once a second
551 if (_lossPrHistory[0].timeMs == -1) {
552 // First, no shift
553 _shortMaxLossPr255 = lossPr255;
554 } else {
555 // Shift
556 for (int32_t i = (kLossPrHistorySize - 2); i >= 0; i--) {
557 _lossPrHistory[i + 1].lossPr255 = _lossPrHistory[i].lossPr255;
558 _lossPrHistory[i + 1].timeMs = _lossPrHistory[i].timeMs;
559 }
560 }
561 if (_shortMaxLossPr255 == 0) {
562 _shortMaxLossPr255 = lossPr255;
563 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000564
philipel9d3ab612015-12-21 04:12:39 -0800565 _lossPrHistory[0].lossPr255 = _shortMaxLossPr255;
566 _lossPrHistory[0].timeMs = now;
567 _shortMaxLossPr255 = 0;
568 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000569}
570
philipel9d3ab612015-12-21 04:12:39 -0800571uint8_t VCMLossProtectionLogic::MaxFilteredLossPr(int64_t nowMs) const {
572 uint8_t maxFound = _shortMaxLossPr255;
573 if (_lossPrHistory[0].timeMs == -1) {
niklase@google.com470e71d2011-07-07 08:21:25 +0000574 return maxFound;
philipel9d3ab612015-12-21 04:12:39 -0800575 }
576 for (int32_t i = 0; i < kLossPrHistorySize; i++) {
577 if (_lossPrHistory[i].timeMs == -1) {
578 break;
579 }
580 if (nowMs - _lossPrHistory[i].timeMs >
581 kLossPrHistorySize * kLossPrShortFilterWinMs) {
582 // This sample (and all samples after this) is too old
583 break;
584 }
585 if (_lossPrHistory[i].lossPr255 > maxFound) {
586 // This sample is the largest one this far into the history
587 maxFound = _lossPrHistory[i].lossPr255;
588 }
589 }
590 return maxFound;
niklase@google.com470e71d2011-07-07 08:21:25 +0000591}
592
philipel9d3ab612015-12-21 04:12:39 -0800593uint8_t VCMLossProtectionLogic::FilteredLoss(int64_t nowMs,
594 FilterPacketLossMode filter_mode,
595 uint8_t lossPr255) {
marpan@webrtc.org2dad3fb2012-01-09 18:18:36 +0000596 // Update the max window filter.
597 UpdateMaxLossHistory(lossPr255, nowMs);
598
599 // Update the recursive average filter.
philipel9d3ab612015-12-21 04:12:39 -0800600 _lossPr255.Apply(static_cast<float>(nowMs - _lastPrUpdateT),
601 static_cast<float>(lossPr255));
marpan@webrtc.org2dad3fb2012-01-09 18:18:36 +0000602 _lastPrUpdateT = nowMs;
603
604 // Filtered loss: default is received loss (no filtering).
pbos@webrtc.org7b859cc2013-04-02 15:54:38 +0000605 uint8_t filtered_loss = lossPr255;
marpan@webrtc.org2dad3fb2012-01-09 18:18:36 +0000606
607 switch (filter_mode) {
608 case kNoFilter:
609 break;
610 case kAvgFilter:
minyue@webrtc.org74aaf292014-07-16 21:28:26 +0000611 filtered_loss = static_cast<uint8_t>(_lossPr255.filtered() + 0.5);
marpan@webrtc.org2dad3fb2012-01-09 18:18:36 +0000612 break;
613 case kMaxFilter:
614 filtered_loss = MaxFilteredLossPr(nowMs);
615 break;
616 }
617
618 return filtered_loss;
niklase@google.com470e71d2011-07-07 08:21:25 +0000619}
620
philipel9d3ab612015-12-21 04:12:39 -0800621void VCMLossProtectionLogic::UpdateFilteredLossPr(uint8_t packetLossEnc) {
622 _lossPr = static_cast<float>(packetLossEnc) / 255.0;
niklase@google.com470e71d2011-07-07 08:21:25 +0000623}
624
philipel9d3ab612015-12-21 04:12:39 -0800625void VCMLossProtectionLogic::UpdateBitRate(float bitRate) {
626 _bitRate = bitRate;
niklase@google.com470e71d2011-07-07 08:21:25 +0000627}
628
philipel9d3ab612015-12-21 04:12:39 -0800629void VCMLossProtectionLogic::UpdatePacketsPerFrame(float nPackets,
630 int64_t nowMs) {
631 _packetsPerFrame.Apply(static_cast<float>(nowMs - _lastPacketPerFrameUpdateT),
632 nPackets);
633 _lastPacketPerFrameUpdateT = nowMs;
niklase@google.com470e71d2011-07-07 08:21:25 +0000634}
635
philipel9d3ab612015-12-21 04:12:39 -0800636void VCMLossProtectionLogic::UpdatePacketsPerFrameKey(float nPackets,
637 int64_t nowMs) {
638 _packetsPerFrameKey.Apply(
639 static_cast<float>(nowMs - _lastPacketPerFrameUpdateTKey), nPackets);
640 _lastPacketPerFrameUpdateTKey = nowMs;
niklase@google.com470e71d2011-07-07 08:21:25 +0000641}
642
philipel9d3ab612015-12-21 04:12:39 -0800643void VCMLossProtectionLogic::UpdateKeyFrameSize(float keyFrameSize) {
644 _keyFrameSize = keyFrameSize;
niklase@google.com470e71d2011-07-07 08:21:25 +0000645}
646
perkjc2c24f72016-07-11 01:47:32 -0700647void VCMLossProtectionLogic::UpdateFrameSize(size_t width, size_t height) {
philipel9d3ab612015-12-21 04:12:39 -0800648 _codecWidth = width;
649 _codecHeight = height;
niklase@google.com470e71d2011-07-07 08:21:25 +0000650}
651
mikhal@webrtc.org0e7d9d82011-12-19 19:04:49 +0000652void VCMLossProtectionLogic::UpdateNumLayers(int numLayers) {
653 _numLayers = (numLayers == 0) ? 1 : numLayers;
654}
655
philipel9d3ab612015-12-21 04:12:39 -0800656bool VCMLossProtectionLogic::UpdateMethod() {
657 if (!_selectedMethod)
658 return false;
659 _currentParameters.rtt = _rtt;
660 _currentParameters.lossPr = _lossPr;
661 _currentParameters.bitRate = _bitRate;
662 _currentParameters.frameRate = _frameRate; // rename actual frame rate?
663 _currentParameters.keyFrameSize = _keyFrameSize;
664 _currentParameters.fecRateDelta = _fecRateDelta;
665 _currentParameters.fecRateKey = _fecRateKey;
666 _currentParameters.packetsPerFrame = _packetsPerFrame.filtered();
667 _currentParameters.packetsPerFrameKey = _packetsPerFrameKey.filtered();
668 _currentParameters.codecWidth = _codecWidth;
669 _currentParameters.codecHeight = _codecHeight;
670 _currentParameters.numLayers = _numLayers;
671 return _selectedMethod->UpdateParameters(&_currentParameters);
niklase@google.com470e71d2011-07-07 08:21:25 +0000672}
673
philipel9d3ab612015-12-21 04:12:39 -0800674VCMProtectionMethod* VCMLossProtectionLogic::SelectedMethod() const {
675 return _selectedMethod.get();
niklase@google.com470e71d2011-07-07 08:21:25 +0000676}
677
pbos@webrtc.orgcade82c2015-03-12 10:39:24 +0000678VCMProtectionMethodEnum VCMLossProtectionLogic::SelectedType() const {
pbosba8c15b2015-07-14 09:36:34 -0700679 return _selectedMethod ? _selectedMethod->Type() : kNone;
mikhal@webrtc.orga057a952011-08-26 21:17:34 +0000680}
681
philipel9d3ab612015-12-21 04:12:39 -0800682void VCMLossProtectionLogic::Reset(int64_t nowMs) {
683 _lastPrUpdateT = nowMs;
684 _lastPacketPerFrameUpdateT = nowMs;
685 _lastPacketPerFrameUpdateTKey = nowMs;
686 _lossPr255.Reset(0.9999f);
687 _packetsPerFrame.Reset(0.9999f);
688 _fecRateDelta = _fecRateKey = 0;
689 for (int32_t i = 0; i < kLossPrHistorySize; i++) {
690 _lossPrHistory[i].lossPr255 = 0;
691 _lossPrHistory[i].timeMs = -1;
692 }
693 _shortMaxLossPr255 = 0;
694 Release();
mikhal@webrtc.orga057a952011-08-26 21:17:34 +0000695}
696
pbosba8c15b2015-07-14 09:36:34 -0700697void VCMLossProtectionLogic::Release() {
698 _selectedMethod.reset();
niklase@google.com470e71d2011-07-07 08:21:25 +0000699}
700
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000701} // namespace media_optimization
702} // namespace webrtc