blob: 436cb0ffdf89e60b43ecb89e7f9252800e23d45e [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:25 +00001/*
stefan@webrtc.orgc35f5ce2012-04-11 07:42:25 +00002 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
niklase@google.com470e71d2011-07-07 08:21:25 +00003 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "modules/video_coding/media_opt_util.h"
niklase@google.com470e71d2011-07-07 08:21:25 +000012
niklase@google.com470e71d2011-07-07 08:21:25 +000013#include <float.h>
14#include <limits.h>
pbos@webrtc.orga4407322013-07-16 12:32:05 +000015#include <math.h>
niklase@google.com470e71d2011-07-07 08:21:25 +000016
philipel9d3ab612015-12-21 04:12:39 -080017#include <algorithm>
brandtr05f845d2016-11-16 22:59:39 -080018#include <limits>
philipel9d3ab612015-12-21 04:12:39 -080019
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020020#include "modules/include/module_common_types.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020021#include "modules/video_coding/fec_rate_table.h"
Yves Gerey665174f2018-06-19 15:03:05 +020022#include "modules/video_coding/include/video_coding_defines.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "modules/video_coding/nack_fec_tables.h"
Sergio Garcia Murillo43800f92018-06-21 16:16:38 +020024#include "modules/video_coding/utility/simulcast_rate_allocator.h"
Emircan Uysaler704a7bd2018-08-06 16:14:10 -070025#include "rtc_base/numerics/safe_conversions.h"
mikhal@webrtc.org0e7d9d82011-12-19 19:04:49 +000026
niklase@google.com470e71d2011-07-07 08:21:25 +000027namespace webrtc {
Peter Boström9cb1f302015-04-01 11:39:49 +020028// Max value of loss rates in off-line model
29static const int kPacketLossMax = 129;
30
stefan@webrtc.orga64300a2013-03-04 15:24:40 +000031namespace media_optimization {
niklase@google.com470e71d2011-07-07 08:21:25 +000032
Stefan Holmerdbdb3a02018-07-17 16:03:46 +020033VCMProtectionParameters::VCMProtectionParameters()
34 : rtt(0),
35 lossPr(0.0f),
36 bitRate(0.0f),
37 packetsPerFrame(0.0f),
38 packetsPerFrameKey(0.0f),
39 frameRate(0.0f),
40 keyFrameSize(0.0f),
41 fecRateDelta(0),
42 fecRateKey(0),
43 codecWidth(0),
44 codecHeight(0),
45 numLayers(1) {}
46
Peter Boström9cb1f302015-04-01 11:39:49 +020047VCMProtectionMethod::VCMProtectionMethod()
48 : _effectivePacketLoss(0),
49 _protectionFactorK(0),
50 _protectionFactorD(0),
51 _scaleProtKey(2.0f),
52 _maxPayloadSize(1460),
Peter Boström9cb1f302015-04-01 11:39:49 +020053 _corrFecCost(1.0),
philipel9d3ab612015-12-21 04:12:39 -080054 _type(kNone) {}
mikhal@webrtc.orga057a952011-08-26 21:17:34 +000055
pbosc0430522016-05-01 17:19:05 -070056VCMProtectionMethod::~VCMProtectionMethod() {}
marpan@google.com86548c62011-07-12 17:12:57 +000057
Stefan Holmerdbdb3a02018-07-17 16:03:46 +020058enum VCMProtectionMethodEnum VCMProtectionMethod::Type() const {
59 return _type;
60}
61
62uint8_t VCMProtectionMethod::RequiredPacketLossER() {
63 return _effectivePacketLoss;
64}
65
66uint8_t VCMProtectionMethod::RequiredProtectionFactorK() {
67 return _protectionFactorK;
68}
69
70uint8_t VCMProtectionMethod::RequiredProtectionFactorD() {
71 return _protectionFactorD;
72}
73
74bool VCMProtectionMethod::RequiredUepProtectionK() {
75 return _useUepProtectionK;
76}
77
78bool VCMProtectionMethod::RequiredUepProtectionD() {
79 return _useUepProtectionD;
80}
81
82int VCMProtectionMethod::MaxFramesFec() const {
83 return 1;
84}
85
pkasting@chromium.org16825b12015-01-12 21:51:21 +000086VCMNackFecMethod::VCMNackFecMethod(int64_t lowRttNackThresholdMs,
87 int64_t highRttNackThresholdMs)
stefan@webrtc.org932ab182011-11-29 11:33:31 +000088 : VCMFecMethod(),
89 _lowRttNackMs(lowRttNackThresholdMs),
stefan@webrtc.orgc35f5ce2012-04-11 07:42:25 +000090 _highRttNackMs(highRttNackThresholdMs),
91 _maxFramesFec(1) {
stefan@webrtc.org932ab182011-11-29 11:33:31 +000092 assert(lowRttNackThresholdMs >= -1 && highRttNackThresholdMs >= -1);
93 assert(highRttNackThresholdMs == -1 ||
94 lowRttNackThresholdMs <= highRttNackThresholdMs);
95 assert(lowRttNackThresholdMs > -1 || highRttNackThresholdMs == -1);
96 _type = kNackFec;
mikhal@google.com77408882011-07-22 22:05:25 +000097}
98
philipel9d3ab612015-12-21 04:12:39 -080099VCMNackFecMethod::~VCMNackFecMethod() {
100 //
mikhal@google.com77408882011-07-22 22:05:25 +0000101}
philipel9d3ab612015-12-21 04:12:39 -0800102bool VCMNackFecMethod::ProtectionFactor(
103 const VCMProtectionParameters* parameters) {
104 // Hybrid Nack FEC has three operational modes:
105 // 1. Low RTT (below kLowRttNackMs) - Nack only: Set FEC rate
106 // (_protectionFactorD) to zero. -1 means no FEC.
107 // 2. High RTT (above _highRttNackMs) - FEC Only: Keep FEC factors.
108 // -1 means always allow NACK.
109 // 3. Medium RTT values - Hybrid mode: We will only nack the
110 // residual following the decoding of the FEC (refer to JB logic). FEC
111 // delta protection factor will be adjusted based on the RTT.
mikhal@google.com022716b2011-07-20 23:12:57 +0000112
philipel9d3ab612015-12-21 04:12:39 -0800113 // Otherwise: we count on FEC; if the RTT is below a threshold, then we
114 // nack the residual, based on a decision made in the JB.
mikhal@google.com022716b2011-07-20 23:12:57 +0000115
philipel9d3ab612015-12-21 04:12:39 -0800116 // Compute the protection factors
117 VCMFecMethod::ProtectionFactor(parameters);
118 if (_lowRttNackMs == -1 || parameters->rtt < _lowRttNackMs) {
119 _protectionFactorD = 0;
120 VCMFecMethod::UpdateProtectionFactorD(_protectionFactorD);
mikhal@google.com320813c2011-08-03 20:47:50 +0000121
mikhal@google.com679450f2011-08-01 22:14:58 +0000122 // When in Hybrid mode (RTT range), adjust FEC rates based on the
123 // RTT (NACK effectiveness) - adjustment factor is in the range [0,1].
philipel9d3ab612015-12-21 04:12:39 -0800124 } else if (_highRttNackMs == -1 || parameters->rtt < _highRttNackMs) {
125 // TODO(mikhal): Disabling adjustment temporarily.
126 // uint16_t rttIndex = (uint16_t) parameters->rtt;
127 float adjustRtt = 1.0f; // (float)VCMNackFecTable[rttIndex] / 100.0f;
niklase@google.com470e71d2011-07-07 08:21:25 +0000128
philipel9d3ab612015-12-21 04:12:39 -0800129 // Adjust FEC with NACK on (for delta frame only)
130 // table depends on RTT relative to rttMax (NACK Threshold)
Emircan Uysaler704a7bd2018-08-06 16:14:10 -0700131 _protectionFactorD = rtc::saturated_cast<uint8_t>(
132 adjustRtt * rtc::saturated_cast<float>(_protectionFactorD));
philipel9d3ab612015-12-21 04:12:39 -0800133 // update FEC rates after applying adjustment
134 VCMFecMethod::UpdateProtectionFactorD(_protectionFactorD);
135 }
mikhal@google.com679450f2011-08-01 22:14:58 +0000136
philipel9d3ab612015-12-21 04:12:39 -0800137 return true;
mikhal@google.com022716b2011-07-20 23:12:57 +0000138}
niklase@google.com470e71d2011-07-07 08:21:25 +0000139
stefan@webrtc.orgc35f5ce2012-04-11 07:42:25 +0000140int VCMNackFecMethod::ComputeMaxFramesFec(
141 const VCMProtectionParameters* parameters) {
142 if (parameters->numLayers > 2) {
143 // For more than 2 temporal layers we will only have FEC on the base layer,
144 // and the base layers will be pretty far apart. Therefore we force one
145 // frame FEC.
146 return 1;
147 }
148 // We set the max number of frames to base the FEC on so that on average
149 // we will have complete frames in one RTT. Note that this is an upper
150 // bound, and that the actual number of frames used for FEC is decided by the
151 // RTP module based on the actual number of packets and the protection factor.
philipel9d3ab612015-12-21 04:12:39 -0800152 float base_layer_framerate =
153 parameters->frameRate /
Emircan Uysaler704a7bd2018-08-06 16:14:10 -0700154 rtc::saturated_cast<float>(1 << (parameters->numLayers - 1));
philipel9d3ab612015-12-21 04:12:39 -0800155 int max_frames_fec = std::max(
Emircan Uysaler704a7bd2018-08-06 16:14:10 -0700156 rtc::saturated_cast<int>(
157 2.0f * base_layer_framerate * parameters->rtt / 1000.0f + 0.5f),
philipel9d3ab612015-12-21 04:12:39 -0800158 1);
stefan@webrtc.orgc35f5ce2012-04-11 07:42:25 +0000159 // |kUpperLimitFramesFec| is the upper limit on how many frames we
160 // allow any FEC to be based on.
161 if (max_frames_fec > kUpperLimitFramesFec) {
162 max_frames_fec = kUpperLimitFramesFec;
163 }
164 return max_frames_fec;
165}
166
167int VCMNackFecMethod::MaxFramesFec() const {
168 return _maxFramesFec;
169}
170
marpan@webrtc.org88ad06b2012-04-20 16:05:24 +0000171bool VCMNackFecMethod::BitRateTooLowForFec(
172 const VCMProtectionParameters* parameters) {
173 // Bitrate below which we turn off FEC, regardless of reported packet loss.
174 // The condition should depend on resolution and content. For now, use
175 // threshold on bytes per frame, with some effect for the frame size.
176 // The condition for turning off FEC is also based on other factors,
177 // such as |_numLayers|, |_maxFramesFec|, and |_rtt|.
178 int estimate_bytes_per_frame = 1000 * BitsPerFrame(parameters) / 8;
179 int max_bytes_per_frame = kMaxBytesPerFrameForFec;
180 int num_pixels = parameters->codecWidth * parameters->codecHeight;
181 if (num_pixels <= 352 * 288) {
182 max_bytes_per_frame = kMaxBytesPerFrameForFecLow;
183 } else if (num_pixels > 640 * 480) {
184 max_bytes_per_frame = kMaxBytesPerFrameForFecHigh;
185 }
philipel9d3ab612015-12-21 04:12:39 -0800186 // TODO(marpan): add condition based on maximum frames used for FEC,
marpan@webrtc.org88ad06b2012-04-20 16:05:24 +0000187 // and expand condition based on frame size.
pkasting@chromium.org16825b12015-01-12 21:51:21 +0000188 // Max round trip time threshold in ms.
189 const int64_t kMaxRttTurnOffFec = 200;
marpan@webrtc.org88ad06b2012-04-20 16:05:24 +0000190 if (estimate_bytes_per_frame < max_bytes_per_frame &&
philipel9d3ab612015-12-21 04:12:39 -0800191 parameters->numLayers < 3 && parameters->rtt < kMaxRttTurnOffFec) {
marpan@webrtc.org88ad06b2012-04-20 16:05:24 +0000192 return true;
193 }
194 return false;
195}
196
philipel9d3ab612015-12-21 04:12:39 -0800197bool VCMNackFecMethod::EffectivePacketLoss(
198 const VCMProtectionParameters* parameters) {
199 // Set the effective packet loss for encoder (based on FEC code).
200 // Compute the effective packet loss and residual packet loss due to FEC.
201 VCMFecMethod::EffectivePacketLoss(parameters);
202 return true;
mikhal@google.com022716b2011-07-20 23:12:57 +0000203}
niklase@google.com470e71d2011-07-07 08:21:25 +0000204
philipel9d3ab612015-12-21 04:12:39 -0800205bool VCMNackFecMethod::UpdateParameters(
206 const VCMProtectionParameters* parameters) {
207 ProtectionFactor(parameters);
208 EffectivePacketLoss(parameters);
209 _maxFramesFec = ComputeMaxFramesFec(parameters);
210 if (BitRateTooLowForFec(parameters)) {
211 _protectionFactorK = 0;
212 _protectionFactorD = 0;
213 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000214
philipel9d3ab612015-12-21 04:12:39 -0800215 // Protection/fec rates obtained above are defined relative to total number
216 // of packets (total rate: source + fec) FEC in RTP module assumes
217 // protection factor is defined relative to source number of packets so we
218 // should convert the factor to reduce mismatch between mediaOpt's rate and
219 // the actual one
220 _protectionFactorK = VCMFecMethod::ConvertFECRate(_protectionFactorK);
221 _protectionFactorD = VCMFecMethod::ConvertFECRate(_protectionFactorD);
niklase@google.com470e71d2011-07-07 08:21:25 +0000222
philipel9d3ab612015-12-21 04:12:39 -0800223 return true;
niklase@google.com470e71d2011-07-07 08:21:25 +0000224}
225
philipel9d3ab612015-12-21 04:12:39 -0800226VCMNackMethod::VCMNackMethod() : VCMProtectionMethod() {
227 _type = kNack;
mikhal@webrtc.orga057a952011-08-26 21:17:34 +0000228}
229
philipel9d3ab612015-12-21 04:12:39 -0800230VCMNackMethod::~VCMNackMethod() {
231 //
mikhal@webrtc.orga057a952011-08-26 21:17:34 +0000232}
233
philipel9d3ab612015-12-21 04:12:39 -0800234bool VCMNackMethod::EffectivePacketLoss(
235 const VCMProtectionParameters* parameter) {
236 // Effective Packet Loss, NA in current version.
237 _effectivePacketLoss = 0;
238 return true;
niklase@google.com470e71d2011-07-07 08:21:25 +0000239}
240
philipel9d3ab612015-12-21 04:12:39 -0800241bool VCMNackMethod::UpdateParameters(
242 const VCMProtectionParameters* parameters) {
243 // Compute the effective packet loss
244 EffectivePacketLoss(parameters);
niklase@google.com470e71d2011-07-07 08:21:25 +0000245
philipel9d3ab612015-12-21 04:12:39 -0800246 // nackCost = (bitRate - nackCost) * (lossPr)
247 return true;
niklase@google.com470e71d2011-07-07 08:21:25 +0000248}
249
philipel9d3ab612015-12-21 04:12:39 -0800250VCMFecMethod::VCMFecMethod() : VCMProtectionMethod() {
251 _type = kFec;
mikhal@webrtc.orga057a952011-08-26 21:17:34 +0000252}
philipel9d3ab612015-12-21 04:12:39 -0800253VCMFecMethod::~VCMFecMethod() {
254 //
mikhal@webrtc.orga057a952011-08-26 21:17:34 +0000255}
256
philipel9d3ab612015-12-21 04:12:39 -0800257uint8_t VCMFecMethod::BoostCodeRateKey(uint8_t packetFrameDelta,
258 uint8_t packetFrameKey) const {
259 uint8_t boostRateKey = 2;
260 // Default: ratio scales the FEC protection up for I frames
261 uint8_t ratio = 1;
niklase@google.com470e71d2011-07-07 08:21:25 +0000262
philipel9d3ab612015-12-21 04:12:39 -0800263 if (packetFrameDelta > 0) {
264 ratio = (int8_t)(packetFrameKey / packetFrameDelta);
265 }
266 ratio = VCM_MAX(boostRateKey, ratio);
niklase@google.com470e71d2011-07-07 08:21:25 +0000267
philipel9d3ab612015-12-21 04:12:39 -0800268 return ratio;
niklase@google.com470e71d2011-07-07 08:21:25 +0000269}
270
philipel9d3ab612015-12-21 04:12:39 -0800271uint8_t VCMFecMethod::ConvertFECRate(uint8_t codeRateRTP) const {
Emircan Uysaler704a7bd2018-08-06 16:14:10 -0700272 return rtc::saturated_cast<uint8_t>(
273 VCM_MIN(255, (0.5 + 255.0 * codeRateRTP /
274 rtc::saturated_cast<float>(255 - codeRateRTP))));
niklase@google.com470e71d2011-07-07 08:21:25 +0000275}
276
mikhal@google.com679450f2011-08-01 22:14:58 +0000277// Update FEC with protectionFactorD
philipel9d3ab612015-12-21 04:12:39 -0800278void VCMFecMethod::UpdateProtectionFactorD(uint8_t protectionFactorD) {
279 _protectionFactorD = protectionFactorD;
mikhal@google.com679450f2011-08-01 22:14:58 +0000280}
281
mikhal@webrtc.orgd0752c32011-10-19 15:48:30 +0000282// Update FEC with protectionFactorK
philipel9d3ab612015-12-21 04:12:39 -0800283void VCMFecMethod::UpdateProtectionFactorK(uint8_t protectionFactorK) {
284 _protectionFactorK = protectionFactorK;
mikhal@webrtc.orgd0752c32011-10-19 15:48:30 +0000285}
286
philipel9d3ab612015-12-21 04:12:39 -0800287bool VCMFecMethod::ProtectionFactor(const VCMProtectionParameters* parameters) {
288 // FEC PROTECTION SETTINGS: varies with packet loss and bitrate
niklase@google.com470e71d2011-07-07 08:21:25 +0000289
philipel9d3ab612015-12-21 04:12:39 -0800290 // No protection if (filtered) packetLoss is 0
Emircan Uysaler704a7bd2018-08-06 16:14:10 -0700291 uint8_t packetLoss = rtc::saturated_cast<uint8_t>(255 * parameters->lossPr);
philipel9d3ab612015-12-21 04:12:39 -0800292 if (packetLoss == 0) {
293 _protectionFactorK = 0;
294 _protectionFactorD = 0;
niklase@google.com470e71d2011-07-07 08:21:25 +0000295 return true;
philipel9d3ab612015-12-21 04:12:39 -0800296 }
297
298 // Parameters for FEC setting:
299 // first partition size, thresholds, table pars, spatial resoln fac.
300
301 // First partition protection: ~ 20%
Emircan Uysaler704a7bd2018-08-06 16:14:10 -0700302 uint8_t firstPartitionProt = rtc::saturated_cast<uint8_t>(255 * 0.20);
philipel9d3ab612015-12-21 04:12:39 -0800303
304 // Minimum protection level needed to generate one FEC packet for one
305 // source packet/frame (in RTP sender)
306 uint8_t minProtLevelFec = 85;
307
308 // Threshold on packetLoss and bitRrate/frameRate (=average #packets),
309 // above which we allocate protection to cover at least first partition.
310 uint8_t lossThr = 0;
311 uint8_t packetNumThr = 1;
312
313 // Parameters for range of rate index of table.
314 const uint8_t ratePar1 = 5;
315 const uint8_t ratePar2 = 49;
316
317 // Spatial resolution size, relative to a reference size.
Emircan Uysaler704a7bd2018-08-06 16:14:10 -0700318 float spatialSizeToRef = rtc::saturated_cast<float>(parameters->codecWidth *
319 parameters->codecHeight) /
320 (rtc::saturated_cast<float>(704 * 576));
philipel9d3ab612015-12-21 04:12:39 -0800321 // resolnFac: This parameter will generally increase/decrease the FEC rate
322 // (for fixed bitRate and packetLoss) based on system size.
323 // Use a smaller exponent (< 1) to control/soften system size effect.
324 const float resolnFac = 1.0 / powf(spatialSizeToRef, 0.3f);
325
326 const int bitRatePerFrame = BitsPerFrame(parameters);
327
328 // Average number of packets per frame (source and fec):
Emircan Uysaler704a7bd2018-08-06 16:14:10 -0700329 const uint8_t avgTotPackets = rtc::saturated_cast<uint8_t>(
330 1.5f + rtc::saturated_cast<float>(bitRatePerFrame) * 1000.0f /
331 rtc::saturated_cast<float>(8.0 * _maxPayloadSize));
philipel9d3ab612015-12-21 04:12:39 -0800332
333 // FEC rate parameters: for P and I frame
334 uint8_t codeRateDelta = 0;
335 uint8_t codeRateKey = 0;
336
337 // Get index for table: the FEC protection depends on an effective rate.
338 // The range on the rate index corresponds to rates (bps)
339 // from ~200k to ~8000k, for 30fps
340 const uint16_t effRateFecTable =
Emircan Uysaler704a7bd2018-08-06 16:14:10 -0700341 rtc::saturated_cast<uint16_t>(resolnFac * bitRatePerFrame);
342 uint8_t rateIndexTable = rtc::saturated_cast<uint8_t>(
brandtr05f845d2016-11-16 22:59:39 -0800343 VCM_MAX(VCM_MIN((effRateFecTable - ratePar1) / ratePar1, ratePar2), 0));
philipel9d3ab612015-12-21 04:12:39 -0800344
345 // Restrict packet loss range to 50:
346 // current tables defined only up to 50%
347 if (packetLoss >= kPacketLossMax) {
348 packetLoss = kPacketLossMax - 1;
349 }
350 uint16_t indexTable = rateIndexTable * kPacketLossMax + packetLoss;
351
352 // Check on table index
brandtr71b1c1f2017-01-12 06:16:24 -0800353 RTC_DCHECK_LT(indexTable, kFecRateTableSize);
philipel9d3ab612015-12-21 04:12:39 -0800354
355 // Protection factor for P frame
brandtr71b1c1f2017-01-12 06:16:24 -0800356 codeRateDelta = kFecRateTable[indexTable];
philipel9d3ab612015-12-21 04:12:39 -0800357
358 if (packetLoss > lossThr && avgTotPackets > packetNumThr) {
359 // Set a minimum based on first partition size.
360 if (codeRateDelta < firstPartitionProt) {
361 codeRateDelta = firstPartitionProt;
362 }
363 }
364
365 // Check limit on amount of protection for P frame; 50% is max.
366 if (codeRateDelta >= kPacketLossMax) {
367 codeRateDelta = kPacketLossMax - 1;
368 }
369
philipel9d3ab612015-12-21 04:12:39 -0800370 // For Key frame:
371 // Effectively at a higher rate, so we scale/boost the rate
372 // The boost factor may depend on several factors: ratio of packet
373 // number of I to P frames, how much protection placed on P frames, etc.
brandtr05f845d2016-11-16 22:59:39 -0800374 const uint8_t packetFrameDelta =
Emircan Uysaler704a7bd2018-08-06 16:14:10 -0700375 rtc::saturated_cast<uint8_t>(0.5 + parameters->packetsPerFrame);
philipel9d3ab612015-12-21 04:12:39 -0800376 const uint8_t packetFrameKey =
Emircan Uysaler704a7bd2018-08-06 16:14:10 -0700377 rtc::saturated_cast<uint8_t>(0.5 + parameters->packetsPerFrameKey);
philipel9d3ab612015-12-21 04:12:39 -0800378 const uint8_t boostKey = BoostCodeRateKey(packetFrameDelta, packetFrameKey);
379
Emircan Uysaler704a7bd2018-08-06 16:14:10 -0700380 rateIndexTable = rtc::saturated_cast<uint8_t>(VCM_MAX(
philipel9d3ab612015-12-21 04:12:39 -0800381 VCM_MIN(1 + (boostKey * effRateFecTable - ratePar1) / ratePar1, ratePar2),
brandtr05f845d2016-11-16 22:59:39 -0800382 0));
philipel9d3ab612015-12-21 04:12:39 -0800383 uint16_t indexTableKey = rateIndexTable * kPacketLossMax + packetLoss;
384
brandtr71b1c1f2017-01-12 06:16:24 -0800385 indexTableKey = VCM_MIN(indexTableKey, kFecRateTableSize);
philipel9d3ab612015-12-21 04:12:39 -0800386
387 // Check on table index
brandtr71b1c1f2017-01-12 06:16:24 -0800388 assert(indexTableKey < kFecRateTableSize);
philipel9d3ab612015-12-21 04:12:39 -0800389
390 // Protection factor for I frame
brandtr71b1c1f2017-01-12 06:16:24 -0800391 codeRateKey = kFecRateTable[indexTableKey];
philipel9d3ab612015-12-21 04:12:39 -0800392
393 // Boosting for Key frame.
394 int boostKeyProt = _scaleProtKey * codeRateDelta;
395 if (boostKeyProt >= kPacketLossMax) {
396 boostKeyProt = kPacketLossMax - 1;
397 }
398
399 // Make sure I frame protection is at least larger than P frame protection,
400 // and at least as high as filtered packet loss.
Emircan Uysaler704a7bd2018-08-06 16:14:10 -0700401 codeRateKey = rtc::saturated_cast<uint8_t>(
philipel9d3ab612015-12-21 04:12:39 -0800402 VCM_MAX(packetLoss, VCM_MAX(boostKeyProt, codeRateKey)));
403
404 // Check limit on amount of protection for I frame: 50% is max.
405 if (codeRateKey >= kPacketLossMax) {
406 codeRateKey = kPacketLossMax - 1;
407 }
408
409 _protectionFactorK = codeRateKey;
410 _protectionFactorD = codeRateDelta;
411
412 // Generally there is a rate mis-match between the FEC cost estimated
413 // in mediaOpt and the actual FEC cost sent out in RTP module.
414 // This is more significant at low rates (small # of source packets), where
415 // the granularity of the FEC decreases. In this case, non-zero protection
416 // in mediaOpt may generate 0 FEC packets in RTP sender (since actual #FEC
417 // is based on rounding off protectionFactor on actual source packet number).
418 // The correction factor (_corrFecCost) attempts to corrects this, at least
419 // for cases of low rates (small #packets) and low protection levels.
420
Emircan Uysaler704a7bd2018-08-06 16:14:10 -0700421 float numPacketsFl =
422 1.0f + (rtc::saturated_cast<float>(bitRatePerFrame) * 1000.0 /
423 rtc::saturated_cast<float>(8.0 * _maxPayloadSize) +
424 0.5);
philipel9d3ab612015-12-21 04:12:39 -0800425
426 const float estNumFecGen =
Emircan Uysaler704a7bd2018-08-06 16:14:10 -0700427 0.5f +
428 rtc::saturated_cast<float>(_protectionFactorD * numPacketsFl / 255.0f);
philipel9d3ab612015-12-21 04:12:39 -0800429
430 // We reduce cost factor (which will reduce overhead for FEC and
431 // hybrid method) and not the protectionFactor.
432 _corrFecCost = 1.0f;
433 if (estNumFecGen < 1.1f && _protectionFactorD < minProtLevelFec) {
434 _corrFecCost = 0.5f;
435 }
436 if (estNumFecGen < 0.9f && _protectionFactorD < minProtLevelFec) {
437 _corrFecCost = 0.0f;
438 }
439
philipel9d3ab612015-12-21 04:12:39 -0800440 // DONE WITH FEC PROTECTION SETTINGS
441 return true;
niklase@google.com470e71d2011-07-07 08:21:25 +0000442}
443
mikhal@webrtc.org0e7d9d82011-12-19 19:04:49 +0000444int VCMFecMethod::BitsPerFrame(const VCMProtectionParameters* parameters) {
445 // When temporal layers are available FEC will only be applied on the base
446 // layer.
Erik Språngd92288f2018-07-04 10:07:40 +0200447 const float bitRateRatio =
448 webrtc::SimulcastRateAllocator::GetTemporalRateAllocation(
449 parameters->numLayers, 0);
thakis@chromium.org65bc2542012-08-13 19:26:12 +0000450 float frameRateRatio = powf(1 / 2.0, parameters->numLayers - 1);
mikhal@webrtc.org0e7d9d82011-12-19 19:04:49 +0000451 float bitRate = parameters->bitRate * bitRateRatio;
452 float frameRate = parameters->frameRate * frameRateRatio;
453
454 // TODO(mikhal): Update factor following testing.
455 float adjustmentFactor = 1;
456
Peter Boström5e8351b2016-01-28 23:55:29 +0100457 if (frameRate < 1.0f)
458 frameRate = 1.0f;
mikhal@webrtc.org0e7d9d82011-12-19 19:04:49 +0000459 // Average bits per frame (units of kbits)
Emircan Uysaler704a7bd2018-08-06 16:14:10 -0700460 return rtc::saturated_cast<int>(adjustmentFactor * bitRate / frameRate);
mikhal@webrtc.org0e7d9d82011-12-19 19:04:49 +0000461}
462
philipel9d3ab612015-12-21 04:12:39 -0800463bool VCMFecMethod::EffectivePacketLoss(
464 const VCMProtectionParameters* parameters) {
465 // Effective packet loss to encoder is based on RPL (residual packet loss)
466 // this is a soft setting based on degree of FEC protection
467 // RPL = received/input packet loss - average_FEC_recovery
468 // note: received/input packet loss may be filtered based on FilteredLoss
niklase@google.com470e71d2011-07-07 08:21:25 +0000469
philipel9d3ab612015-12-21 04:12:39 -0800470 // Effective Packet Loss, NA in current version.
471 _effectivePacketLoss = 0;
niklase@google.com470e71d2011-07-07 08:21:25 +0000472
philipel9d3ab612015-12-21 04:12:39 -0800473 return true;
niklase@google.com470e71d2011-07-07 08:21:25 +0000474}
475
philipel9d3ab612015-12-21 04:12:39 -0800476bool VCMFecMethod::UpdateParameters(const VCMProtectionParameters* parameters) {
477 // Compute the protection factor
478 ProtectionFactor(parameters);
niklase@google.com470e71d2011-07-07 08:21:25 +0000479
philipel9d3ab612015-12-21 04:12:39 -0800480 // Compute the effective packet loss
481 EffectivePacketLoss(parameters);
niklase@google.com470e71d2011-07-07 08:21:25 +0000482
philipel9d3ab612015-12-21 04:12:39 -0800483 // Protection/fec rates obtained above is defined relative to total number
484 // of packets (total rate: source+fec) FEC in RTP module assumes protection
485 // factor is defined relative to source number of packets so we should
486 // convert the factor to reduce mismatch between mediaOpt suggested rate and
487 // the actual rate
488 _protectionFactorK = ConvertFECRate(_protectionFactorK);
489 _protectionFactorD = ConvertFECRate(_protectionFactorD);
niklase@google.com470e71d2011-07-07 08:21:25 +0000490
philipel9d3ab612015-12-21 04:12:39 -0800491 return true;
niklase@google.com470e71d2011-07-07 08:21:25 +0000492}
philipel9d3ab612015-12-21 04:12:39 -0800493VCMLossProtectionLogic::VCMLossProtectionLogic(int64_t nowMs)
494 : _currentParameters(),
495 _rtt(0),
496 _lossPr(0.0f),
497 _bitRate(0.0f),
498 _frameRate(0.0f),
499 _keyFrameSize(0.0f),
500 _fecRateKey(0),
501 _fecRateDelta(0),
502 _lastPrUpdateT(0),
503 _lossPr255(0.9999f),
504 _lossPrHistory(),
505 _shortMaxLossPr255(0),
506 _packetsPerFrame(0.9999f),
507 _packetsPerFrameKey(0.9999f),
Ying Wang1f262cc2018-08-23 13:54:08 +0200508 _codecWidth(704),
509 _codecHeight(576),
philipel9d3ab612015-12-21 04:12:39 -0800510 _numLayers(1) {
511 Reset(nowMs);
niklase@google.com470e71d2011-07-07 08:21:25 +0000512}
513
philipel9d3ab612015-12-21 04:12:39 -0800514VCMLossProtectionLogic::~VCMLossProtectionLogic() {
515 Release();
niklase@google.com470e71d2011-07-07 08:21:25 +0000516}
517
pbos@webrtc.orgcade82c2015-03-12 10:39:24 +0000518void VCMLossProtectionLogic::SetMethod(
519 enum VCMProtectionMethodEnum newMethodType) {
pbosba8c15b2015-07-14 09:36:34 -0700520 if (_selectedMethod && _selectedMethod->Type() == newMethodType)
521 return;
mikhal@webrtc.orga057a952011-08-26 21:17:34 +0000522
philipel9d3ab612015-12-21 04:12:39 -0800523 switch (newMethodType) {
pbos@webrtc.orgcade82c2015-03-12 10:39:24 +0000524 case kNack:
pbosba8c15b2015-07-14 09:36:34 -0700525 _selectedMethod.reset(new VCMNackMethod());
pbos@webrtc.orgcade82c2015-03-12 10:39:24 +0000526 break;
527 case kFec:
pbosba8c15b2015-07-14 09:36:34 -0700528 _selectedMethod.reset(new VCMFecMethod());
pbos@webrtc.orgcade82c2015-03-12 10:39:24 +0000529 break;
530 case kNackFec:
pbosba8c15b2015-07-14 09:36:34 -0700531 _selectedMethod.reset(new VCMNackFecMethod(kLowRttNackMs, -1));
pbos@webrtc.orgcade82c2015-03-12 10:39:24 +0000532 break;
533 case kNone:
pbosba8c15b2015-07-14 09:36:34 -0700534 _selectedMethod.reset();
pbos@webrtc.orgcade82c2015-03-12 10:39:24 +0000535 break;
536 }
537 UpdateMethod();
niklase@google.com470e71d2011-07-07 08:21:25 +0000538}
539
philipel9d3ab612015-12-21 04:12:39 -0800540void VCMLossProtectionLogic::UpdateRtt(int64_t rtt) {
541 _rtt = rtt;
niklase@google.com470e71d2011-07-07 08:21:25 +0000542}
543
philipel9d3ab612015-12-21 04:12:39 -0800544void VCMLossProtectionLogic::UpdateMaxLossHistory(uint8_t lossPr255,
545 int64_t now) {
546 if (_lossPrHistory[0].timeMs >= 0 &&
547 now - _lossPrHistory[0].timeMs < kLossPrShortFilterWinMs) {
548 if (lossPr255 > _shortMaxLossPr255) {
549 _shortMaxLossPr255 = lossPr255;
niklase@google.com470e71d2011-07-07 08:21:25 +0000550 }
philipel9d3ab612015-12-21 04:12:39 -0800551 } else {
552 // Only add a new value to the history once a second
553 if (_lossPrHistory[0].timeMs == -1) {
554 // First, no shift
555 _shortMaxLossPr255 = lossPr255;
556 } else {
557 // Shift
558 for (int32_t i = (kLossPrHistorySize - 2); i >= 0; i--) {
559 _lossPrHistory[i + 1].lossPr255 = _lossPrHistory[i].lossPr255;
560 _lossPrHistory[i + 1].timeMs = _lossPrHistory[i].timeMs;
561 }
562 }
563 if (_shortMaxLossPr255 == 0) {
564 _shortMaxLossPr255 = lossPr255;
565 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000566
philipel9d3ab612015-12-21 04:12:39 -0800567 _lossPrHistory[0].lossPr255 = _shortMaxLossPr255;
568 _lossPrHistory[0].timeMs = now;
569 _shortMaxLossPr255 = 0;
570 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000571}
572
philipel9d3ab612015-12-21 04:12:39 -0800573uint8_t VCMLossProtectionLogic::MaxFilteredLossPr(int64_t nowMs) const {
574 uint8_t maxFound = _shortMaxLossPr255;
575 if (_lossPrHistory[0].timeMs == -1) {
niklase@google.com470e71d2011-07-07 08:21:25 +0000576 return maxFound;
philipel9d3ab612015-12-21 04:12:39 -0800577 }
578 for (int32_t i = 0; i < kLossPrHistorySize; i++) {
579 if (_lossPrHistory[i].timeMs == -1) {
580 break;
581 }
582 if (nowMs - _lossPrHistory[i].timeMs >
583 kLossPrHistorySize * kLossPrShortFilterWinMs) {
584 // This sample (and all samples after this) is too old
585 break;
586 }
587 if (_lossPrHistory[i].lossPr255 > maxFound) {
588 // This sample is the largest one this far into the history
589 maxFound = _lossPrHistory[i].lossPr255;
590 }
591 }
592 return maxFound;
niklase@google.com470e71d2011-07-07 08:21:25 +0000593}
594
philipel9d3ab612015-12-21 04:12:39 -0800595uint8_t VCMLossProtectionLogic::FilteredLoss(int64_t nowMs,
596 FilterPacketLossMode filter_mode,
597 uint8_t lossPr255) {
marpan@webrtc.org2dad3fb2012-01-09 18:18:36 +0000598 // Update the max window filter.
599 UpdateMaxLossHistory(lossPr255, nowMs);
600
601 // Update the recursive average filter.
Emircan Uysaler704a7bd2018-08-06 16:14:10 -0700602 _lossPr255.Apply(rtc::saturated_cast<float>(nowMs - _lastPrUpdateT),
603 rtc::saturated_cast<float>(lossPr255));
marpan@webrtc.org2dad3fb2012-01-09 18:18:36 +0000604 _lastPrUpdateT = nowMs;
605
606 // Filtered loss: default is received loss (no filtering).
pbos@webrtc.org7b859cc2013-04-02 15:54:38 +0000607 uint8_t filtered_loss = lossPr255;
marpan@webrtc.org2dad3fb2012-01-09 18:18:36 +0000608
609 switch (filter_mode) {
610 case kNoFilter:
611 break;
612 case kAvgFilter:
Emircan Uysaler704a7bd2018-08-06 16:14:10 -0700613 filtered_loss = rtc::saturated_cast<uint8_t>(_lossPr255.filtered() + 0.5);
marpan@webrtc.org2dad3fb2012-01-09 18:18:36 +0000614 break;
615 case kMaxFilter:
616 filtered_loss = MaxFilteredLossPr(nowMs);
617 break;
618 }
619
620 return filtered_loss;
niklase@google.com470e71d2011-07-07 08:21:25 +0000621}
622
philipel9d3ab612015-12-21 04:12:39 -0800623void VCMLossProtectionLogic::UpdateFilteredLossPr(uint8_t packetLossEnc) {
Emircan Uysaler704a7bd2018-08-06 16:14:10 -0700624 _lossPr = rtc::saturated_cast<float>(packetLossEnc) / 255.0;
niklase@google.com470e71d2011-07-07 08:21:25 +0000625}
626
philipel9d3ab612015-12-21 04:12:39 -0800627void VCMLossProtectionLogic::UpdateBitRate(float bitRate) {
628 _bitRate = bitRate;
niklase@google.com470e71d2011-07-07 08:21:25 +0000629}
630
philipel9d3ab612015-12-21 04:12:39 -0800631void VCMLossProtectionLogic::UpdatePacketsPerFrame(float nPackets,
632 int64_t nowMs) {
Emircan Uysaler704a7bd2018-08-06 16:14:10 -0700633 _packetsPerFrame.Apply(
634 rtc::saturated_cast<float>(nowMs - _lastPacketPerFrameUpdateT), nPackets);
philipel9d3ab612015-12-21 04:12:39 -0800635 _lastPacketPerFrameUpdateT = nowMs;
niklase@google.com470e71d2011-07-07 08:21:25 +0000636}
637
philipel9d3ab612015-12-21 04:12:39 -0800638void VCMLossProtectionLogic::UpdatePacketsPerFrameKey(float nPackets,
639 int64_t nowMs) {
640 _packetsPerFrameKey.Apply(
Emircan Uysaler704a7bd2018-08-06 16:14:10 -0700641 rtc::saturated_cast<float>(nowMs - _lastPacketPerFrameUpdateTKey),
642 nPackets);
philipel9d3ab612015-12-21 04:12:39 -0800643 _lastPacketPerFrameUpdateTKey = nowMs;
niklase@google.com470e71d2011-07-07 08:21:25 +0000644}
645
philipel9d3ab612015-12-21 04:12:39 -0800646void VCMLossProtectionLogic::UpdateKeyFrameSize(float keyFrameSize) {
647 _keyFrameSize = keyFrameSize;
niklase@google.com470e71d2011-07-07 08:21:25 +0000648}
649
perkjc2c24f72016-07-11 01:47:32 -0700650void VCMLossProtectionLogic::UpdateFrameSize(size_t width, size_t height) {
philipel9d3ab612015-12-21 04:12:39 -0800651 _codecWidth = width;
652 _codecHeight = height;
niklase@google.com470e71d2011-07-07 08:21:25 +0000653}
654
mikhal@webrtc.org0e7d9d82011-12-19 19:04:49 +0000655void VCMLossProtectionLogic::UpdateNumLayers(int numLayers) {
656 _numLayers = (numLayers == 0) ? 1 : numLayers;
657}
658
philipel9d3ab612015-12-21 04:12:39 -0800659bool VCMLossProtectionLogic::UpdateMethod() {
660 if (!_selectedMethod)
661 return false;
662 _currentParameters.rtt = _rtt;
663 _currentParameters.lossPr = _lossPr;
664 _currentParameters.bitRate = _bitRate;
665 _currentParameters.frameRate = _frameRate; // rename actual frame rate?
666 _currentParameters.keyFrameSize = _keyFrameSize;
667 _currentParameters.fecRateDelta = _fecRateDelta;
668 _currentParameters.fecRateKey = _fecRateKey;
669 _currentParameters.packetsPerFrame = _packetsPerFrame.filtered();
670 _currentParameters.packetsPerFrameKey = _packetsPerFrameKey.filtered();
671 _currentParameters.codecWidth = _codecWidth;
672 _currentParameters.codecHeight = _codecHeight;
673 _currentParameters.numLayers = _numLayers;
674 return _selectedMethod->UpdateParameters(&_currentParameters);
niklase@google.com470e71d2011-07-07 08:21:25 +0000675}
676
philipel9d3ab612015-12-21 04:12:39 -0800677VCMProtectionMethod* VCMLossProtectionLogic::SelectedMethod() const {
678 return _selectedMethod.get();
niklase@google.com470e71d2011-07-07 08:21:25 +0000679}
680
pbos@webrtc.orgcade82c2015-03-12 10:39:24 +0000681VCMProtectionMethodEnum VCMLossProtectionLogic::SelectedType() const {
pbosba8c15b2015-07-14 09:36:34 -0700682 return _selectedMethod ? _selectedMethod->Type() : kNone;
mikhal@webrtc.orga057a952011-08-26 21:17:34 +0000683}
684
philipel9d3ab612015-12-21 04:12:39 -0800685void VCMLossProtectionLogic::Reset(int64_t nowMs) {
686 _lastPrUpdateT = nowMs;
687 _lastPacketPerFrameUpdateT = nowMs;
688 _lastPacketPerFrameUpdateTKey = nowMs;
689 _lossPr255.Reset(0.9999f);
690 _packetsPerFrame.Reset(0.9999f);
691 _fecRateDelta = _fecRateKey = 0;
692 for (int32_t i = 0; i < kLossPrHistorySize; i++) {
693 _lossPrHistory[i].lossPr255 = 0;
694 _lossPrHistory[i].timeMs = -1;
695 }
696 _shortMaxLossPr255 = 0;
697 Release();
mikhal@webrtc.orga057a952011-08-26 21:17:34 +0000698}
699
pbosba8c15b2015-07-14 09:36:34 -0700700void VCMLossProtectionLogic::Release() {
701 _selectedMethod.reset();
niklase@google.com470e71d2011-07-07 08:21:25 +0000702}
703
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000704} // namespace media_optimization
705} // namespace webrtc