blob: dbd315fcf15a1e6a0a5a0ce8d29e7725f896d6b8 [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:25 +00001/*
stefan@webrtc.org29794612012-02-08 08:58:55 +00002 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
niklase@google.com470e71d2011-07-07 08:21:25 +00003 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020010#include "modules/video_coding/jitter_buffer.h"
niklase@google.com470e71d2011-07-07 08:21:25 +000011
pbos@webrtc.org12dc1a32013-08-05 16:22:53 +000012#include <assert.h>
13
stefan@webrtc.org29794612012-02-08 08:58:55 +000014#include <algorithm>
philipel83f831a2016-03-12 03:30:23 -080015#include <limits>
agalusza@google.comd818dcb2013-07-29 21:48:11 +000016#include <utility>
stefan@webrtc.org29794612012-02-08 08:58:55 +000017
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020018#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
19#include "modules/video_coding/frame_buffer.h"
20#include "modules/video_coding/include/video_coding.h"
21#include "modules/video_coding/inter_frame_delay.h"
22#include "modules/video_coding/internal_defines.h"
23#include "modules/video_coding/jitter_buffer_common.h"
24#include "modules/video_coding/jitter_estimator.h"
25#include "modules/video_coding/packet.h"
26#include "rtc_base/checks.h"
27#include "rtc_base/logging.h"
Karl Wiberg80ba3332018-02-05 10:33:35 +010028#include "rtc_base/system/fallthrough.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020029#include "system_wrappers/include/clock.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020030#include "system_wrappers/include/field_trial.h"
niklase@google.com470e71d2011-07-07 08:21:25 +000031
niklase@google.com470e71d2011-07-07 08:21:25 +000032namespace webrtc {
asapersson9a4cd872015-10-23 00:27:14 -070033// Interval for updating SS data.
34static const uint32_t kSsCleanupIntervalSec = 60;
35
mflodman@webrtc.org7c894b72012-11-26 12:40:15 +000036// Use this rtt if no value has been reported.
pkasting@chromium.org16825b12015-01-12 21:51:21 +000037static const int64_t kDefaultRtt = 200;
mflodman@webrtc.org7c894b72012-11-26 12:40:15 +000038
jbauchdb81ffd2015-11-23 03:59:02 -080039// Request a keyframe if no continuous frame has been received for this
40// number of milliseconds and NACKs are disabled.
41static const int64_t kMaxDiscontinuousFramesTime = 1000;
42
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +000043typedef std::pair<uint32_t, VCMFrameBuffer*> FrameListPair;
niklase@google.com470e71d2011-07-07 08:21:25 +000044
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +000045bool IsKeyFrame(FrameListPair pair) {
Niels Möller8f7ce222019-03-21 15:43:58 +010046 return pair.second->FrameType() == VideoFrameType::kVideoFrameKey;
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +000047}
stefan@webrtc.org29794612012-02-08 08:58:55 +000048
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +000049bool HasNonEmptyState(FrameListPair pair) {
50 return pair.second->GetState() != kStateEmpty;
stefan@webrtc.orga64300a2013-03-04 15:24:40 +000051}
52
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +000053void FrameList::InsertFrame(VCMFrameBuffer* frame) {
Niels Möller23775882018-08-16 10:24:12 +020054 insert(rbegin().base(), FrameListPair(frame->Timestamp(), frame));
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +000055}
56
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +000057VCMFrameBuffer* FrameList::PopFrame(uint32_t timestamp) {
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +000058 FrameList::iterator it = find(timestamp);
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +000059 if (it == end())
60 return NULL;
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +000061 VCMFrameBuffer* frame = it->second;
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +000062 erase(it);
63 return frame;
64}
65
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +000066VCMFrameBuffer* FrameList::Front() const {
67 return begin()->second;
68}
69
70VCMFrameBuffer* FrameList::Back() const {
71 return rbegin()->second;
72}
73
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +000074int FrameList::RecycleFramesUntilKeyFrame(FrameList::iterator* key_frame_it,
75 UnorderedFrameList* free_frames) {
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +000076 int drop_count = 0;
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +000077 FrameList::iterator it = begin();
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +000078 while (!empty()) {
79 // Throw at least one frame.
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +000080 it->second->Reset();
81 free_frames->push_back(it->second);
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +000082 erase(it++);
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +000083 ++drop_count;
Niels Möller8f7ce222019-03-21 15:43:58 +010084 if (it != end() &&
85 it->second->FrameType() == VideoFrameType::kVideoFrameKey) {
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +000086 *key_frame_it = it;
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +000087 return drop_count;
88 }
89 }
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +000090 *key_frame_it = end();
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +000091 return drop_count;
92}
93
pbos@webrtc.org4f16c872014-11-24 09:06:48 +000094void FrameList::CleanUpOldOrEmptyFrames(VCMDecodingState* decoding_state,
philipel9d3ab612015-12-21 04:12:39 -080095 UnorderedFrameList* free_frames) {
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +000096 while (!empty()) {
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +000097 VCMFrameBuffer* oldest_frame = Front();
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +000098 bool remove_frame = false;
99 if (oldest_frame->GetState() == kStateEmpty && size() > 1) {
100 // This frame is empty, try to update the last decoded state and drop it
101 // if successful.
102 remove_frame = decoding_state->UpdateEmptyFrame(oldest_frame);
103 } else {
104 remove_frame = decoding_state->IsOldFrame(oldest_frame);
105 }
106 if (!remove_frame) {
107 break;
108 }
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +0000109 free_frames->push_back(oldest_frame);
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000110 erase(begin());
111 }
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000112}
113
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +0000114void FrameList::Reset(UnorderedFrameList* free_frames) {
115 while (!empty()) {
116 begin()->second->Reset();
117 free_frames->push_back(begin()->second);
118 erase(begin());
119 }
120}
121
Stefan Holmerdbdb3a02018-07-17 16:03:46 +0200122Vp9SsMap::Vp9SsMap() {}
123Vp9SsMap::~Vp9SsMap() {}
124
asapersson9a4cd872015-10-23 00:27:14 -0700125bool Vp9SsMap::Insert(const VCMPacket& packet) {
philipel29d88462018-08-08 14:26:00 +0200126 const auto& vp9_header =
127 absl::get<RTPVideoHeaderVP9>(packet.video_header.video_type_header);
128 if (!vp9_header.ss_data_available)
asapersson9a4cd872015-10-23 00:27:14 -0700129 return false;
130
philipel29d88462018-08-08 14:26:00 +0200131 ss_map_[packet.timestamp] = vp9_header.gof;
asapersson9a4cd872015-10-23 00:27:14 -0700132 return true;
133}
134
135void Vp9SsMap::Reset() {
136 ss_map_.clear();
137}
138
139bool Vp9SsMap::Find(uint32_t timestamp, SsMap::iterator* it_out) {
140 bool found = false;
141 for (SsMap::iterator it = ss_map_.begin(); it != ss_map_.end(); ++it) {
142 if (it->first == timestamp || IsNewerTimestamp(timestamp, it->first)) {
143 *it_out = it;
144 found = true;
145 }
146 }
147 return found;
148}
149
150void Vp9SsMap::RemoveOld(uint32_t timestamp) {
151 if (!TimeForCleanup(timestamp))
152 return;
153
154 SsMap::iterator it;
155 if (!Find(timestamp, &it))
156 return;
157
158 ss_map_.erase(ss_map_.begin(), it);
159 AdvanceFront(timestamp);
160}
161
162bool Vp9SsMap::TimeForCleanup(uint32_t timestamp) const {
163 if (ss_map_.empty() || !IsNewerTimestamp(timestamp, ss_map_.begin()->first))
164 return false;
165
166 uint32_t diff = timestamp - ss_map_.begin()->first;
167 return diff / kVideoPayloadTypeFrequency >= kSsCleanupIntervalSec;
168}
169
170void Vp9SsMap::AdvanceFront(uint32_t timestamp) {
171 RTC_DCHECK(!ss_map_.empty());
172 GofInfoVP9 gof = ss_map_.begin()->second;
173 ss_map_.erase(ss_map_.begin());
174 ss_map_[timestamp] = gof;
175}
176
asaperssonc253a1c2015-11-06 00:12:01 -0800177// TODO(asapersson): Update according to updates in RTP payload profile.
asapersson9a4cd872015-10-23 00:27:14 -0700178bool Vp9SsMap::UpdatePacket(VCMPacket* packet) {
philipel29d88462018-08-08 14:26:00 +0200179 auto& vp9_header =
180 absl::get<RTPVideoHeaderVP9>(packet->video_header.video_type_header);
181 uint8_t gof_idx = vp9_header.gof_idx;
asapersson9a4cd872015-10-23 00:27:14 -0700182 if (gof_idx == kNoGofIdx)
183 return false; // No update needed.
184
185 SsMap::iterator it;
186 if (!Find(packet->timestamp, &it))
187 return false; // Corresponding SS not yet received.
188
189 if (gof_idx >= it->second.num_frames_in_gof)
190 return false; // Assume corresponding SS not yet received.
191
philipel29d88462018-08-08 14:26:00 +0200192 vp9_header.temporal_idx = it->second.temporal_idx[gof_idx];
193 vp9_header.temporal_up_switch = it->second.temporal_up_switch[gof_idx];
asapersson9a4cd872015-10-23 00:27:14 -0700194
195 // TODO(asapersson): Set vp9.ref_picture_id[i] and add usage.
philipel29d88462018-08-08 14:26:00 +0200196 vp9_header.num_ref_pics = it->second.num_ref_pics[gof_idx];
asaperssonc253a1c2015-11-06 00:12:01 -0800197 for (uint8_t i = 0; i < it->second.num_ref_pics[gof_idx]; ++i) {
philipel29d88462018-08-08 14:26:00 +0200198 vp9_header.pid_diff[i] = it->second.pid_diff[gof_idx][i];
asapersson9a4cd872015-10-23 00:27:14 -0700199 }
200 return true;
201}
202
203void Vp9SsMap::UpdateFrames(FrameList* frames) {
204 for (const auto& frame_it : *frames) {
205 uint8_t gof_idx =
206 frame_it.second->CodecSpecific()->codecSpecific.VP9.gof_idx;
207 if (gof_idx == kNoGofIdx) {
208 continue;
209 }
210 SsMap::iterator ss_it;
Niels Möller23775882018-08-16 10:24:12 +0200211 if (Find(frame_it.second->Timestamp(), &ss_it)) {
asapersson9a4cd872015-10-23 00:27:14 -0700212 if (gof_idx >= ss_it->second.num_frames_in_gof) {
213 continue; // Assume corresponding SS not yet received.
214 }
215 frame_it.second->SetGofInfo(ss_it->second, gof_idx);
216 }
217 }
218}
219
Qiang Chend4cec152015-06-19 09:17:00 -0700220VCMJitterBuffer::VCMJitterBuffer(Clock* clock,
Niels Möllerdb64d992019-03-29 14:30:53 +0100221 std::unique_ptr<EventWrapper> event)
stefan@webrtc.org34c5da62014-04-11 14:08:35 +0000222 : clock_(clock),
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000223 running_(false),
kwiberg0eb15ed2015-12-17 03:04:15 -0800224 frame_event_(std::move(event)),
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000225 max_number_of_frames_(kStartNumberOfFrames),
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +0000226 free_frames_(),
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000227 decodable_frames_(),
228 incomplete_frames_(),
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000229 last_decoded_state_(),
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000230 first_packet_since_reset_(true),
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000231 incoming_frame_rate_(0),
232 incoming_frame_count_(0),
233 time_last_incoming_frame_count_(0),
234 incoming_bit_count_(0),
235 incoming_bit_rate_(0),
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000236 num_consecutive_old_packets_(0),
asapersson@webrtc.org96dc6852014-11-03 14:40:38 +0000237 num_packets_(0),
238 num_duplicated_packets_(0),
sprang@webrtc.org70e2d112014-09-24 14:06:56 +0000239 jitter_estimate_(clock),
stefan@webrtc.orga678a3b2013-01-21 07:42:11 +0000240 inter_frame_delay_(clock_->TimeInMilliseconds()),
mflodman@webrtc.org7c894b72012-11-26 12:40:15 +0000241 rtt_ms_(kDefaultRtt),
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000242 nack_mode_(kNoNack),
243 low_rtt_nack_threshold_ms_(-1),
244 high_rtt_nack_threshold_ms_(-1),
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000245 missing_sequence_numbers_(SequenceNumberLessThan()),
sprang22691e02016-07-13 10:57:07 -0700246 latest_received_sequence_number_(0),
stefan@webrtc.orgbecf9c82013-02-01 15:09:57 +0000247 max_nack_list_size_(0),
248 max_packet_age_to_nack_(0),
stefan@webrtc.orgef144882013-05-07 19:16:33 +0000249 max_incomplete_time_ms_(0),
agalusza@google.comd818dcb2013-07-29 21:48:11 +0000250 average_packets_per_frame_(0.0f),
philipel85130292016-07-06 16:10:31 +0200251 frame_counter_(0) {
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000252 for (int i = 0; i < kStartNumberOfFrames; i++)
253 free_frames_.push_back(new VCMFrameBuffer());
niklase@google.com470e71d2011-07-07 08:21:25 +0000254}
255
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000256VCMJitterBuffer::~VCMJitterBuffer() {
257 Stop();
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000258 for (UnorderedFrameList::iterator it = free_frames_.begin();
259 it != free_frames_.end(); ++it) {
260 delete *it;
261 }
262 for (FrameList::iterator it = incomplete_frames_.begin();
263 it != incomplete_frames_.end(); ++it) {
264 delete it->second;
265 }
266 for (FrameList::iterator it = decodable_frames_.begin();
267 it != decodable_frames_.end(); ++it) {
268 delete it->second;
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000269 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000270}
271
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000272void VCMJitterBuffer::Start() {
kthelgasonff046c72017-03-31 02:03:55 -0700273 rtc::CritScope cs(&crit_sect_);
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000274 running_ = true;
275 incoming_frame_count_ = 0;
276 incoming_frame_rate_ = 0;
277 incoming_bit_count_ = 0;
278 incoming_bit_rate_ = 0;
stefan@webrtc.orga678a3b2013-01-21 07:42:11 +0000279 time_last_incoming_frame_count_ = clock_->TimeInMilliseconds();
niklase@google.com470e71d2011-07-07 08:21:25 +0000280
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000281 num_consecutive_old_packets_ = 0;
asapersson@webrtc.org96dc6852014-11-03 14:40:38 +0000282 num_packets_ = 0;
283 num_duplicated_packets_ = 0;
niklase@google.com470e71d2011-07-07 08:21:25 +0000284
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000285 // Start in a non-signaled state.
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000286 waiting_for_completion_.frame_size = 0;
287 waiting_for_completion_.timestamp = 0;
288 waiting_for_completion_.latest_packet_time = -1;
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000289 first_packet_since_reset_ = true;
mflodman@webrtc.org7c894b72012-11-26 12:40:15 +0000290 rtt_ms_ = kDefaultRtt;
mikhal@webrtc.org8392cd92013-04-25 21:30:50 +0000291 last_decoded_state_.Reset();
sprang22691e02016-07-13 10:57:07 -0700292
293 decodable_frames_.Reset(&free_frames_);
294 incomplete_frames_.Reset(&free_frames_);
niklase@google.com470e71d2011-07-07 08:21:25 +0000295}
296
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000297void VCMJitterBuffer::Stop() {
kthelgasonff046c72017-03-31 02:03:55 -0700298 rtc::CritScope cs(&crit_sect_);
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000299 running_ = false;
300 last_decoded_state_.Reset();
asaperssona9455ab2015-07-31 06:10:09 -0700301
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000302 // Make sure we wake up any threads waiting on these events.
stefan@webrtc.org2baf5f52013-03-13 08:46:25 +0000303 frame_event_->Set();
niklase@google.com470e71d2011-07-07 08:21:25 +0000304}
305
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000306bool VCMJitterBuffer::Running() const {
kthelgasonff046c72017-03-31 02:03:55 -0700307 rtc::CritScope cs(&crit_sect_);
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000308 return running_;
niklase@google.com470e71d2011-07-07 08:21:25 +0000309}
310
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000311void VCMJitterBuffer::Flush() {
kthelgasonff046c72017-03-31 02:03:55 -0700312 rtc::CritScope cs(&crit_sect_);
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +0000313 decodable_frames_.Reset(&free_frames_);
314 incomplete_frames_.Reset(&free_frames_);
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000315 last_decoded_state_.Reset(); // TODO(mikhal): sync reset.
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000316 num_consecutive_old_packets_ = 0;
317 // Also reset the jitter and delay estimates
318 jitter_estimate_.Reset();
stefan@webrtc.orga678a3b2013-01-21 07:42:11 +0000319 inter_frame_delay_.Reset(clock_->TimeInMilliseconds());
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000320 waiting_for_completion_.frame_size = 0;
321 waiting_for_completion_.timestamp = 0;
322 waiting_for_completion_.latest_packet_time = -1;
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000323 first_packet_since_reset_ = true;
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000324 missing_sequence_numbers_.clear();
niklase@google.com470e71d2011-07-07 08:21:25 +0000325}
326
asapersson@webrtc.org96dc6852014-11-03 14:40:38 +0000327int VCMJitterBuffer::num_packets() const {
kthelgasonff046c72017-03-31 02:03:55 -0700328 rtc::CritScope cs(&crit_sect_);
asapersson@webrtc.org96dc6852014-11-03 14:40:38 +0000329 return num_packets_;
330}
331
332int VCMJitterBuffer::num_duplicated_packets() const {
kthelgasonff046c72017-03-31 02:03:55 -0700333 rtc::CritScope cs(&crit_sect_);
asapersson@webrtc.org96dc6852014-11-03 14:40:38 +0000334 return num_duplicated_packets_;
335}
336
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000337// Calculate framerate and bitrate.
338void VCMJitterBuffer::IncomingRateStatistics(unsigned int* framerate,
339 unsigned int* bitrate) {
340 assert(framerate);
341 assert(bitrate);
kthelgasonff046c72017-03-31 02:03:55 -0700342 rtc::CritScope cs(&crit_sect_);
stefan@webrtc.orga678a3b2013-01-21 07:42:11 +0000343 const int64_t now = clock_->TimeInMilliseconds();
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000344 int64_t diff = now - time_last_incoming_frame_count_;
345 if (diff < 1000 && incoming_frame_rate_ > 0 && incoming_bit_rate_ > 0) {
346 // Make sure we report something even though less than
347 // 1 second has passed since last update.
348 *framerate = incoming_frame_rate_;
349 *bitrate = incoming_bit_rate_;
350 } else if (incoming_frame_count_ != 0) {
351 // We have received frame(s) since last call to this function
352
353 // Prepare calculations
354 if (diff <= 0) {
355 diff = 1;
niklase@google.com470e71d2011-07-07 08:21:25 +0000356 }
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000357 // we add 0.5f for rounding
358 float rate = 0.5f + ((incoming_frame_count_ * 1000.0f) / diff);
359 if (rate < 1.0f) {
360 rate = 1.0f;
361 }
362
363 // Calculate frame rate
364 // Let r be rate.
365 // r(0) = 1000*framecount/delta_time.
366 // (I.e. frames per second since last calculation.)
367 // frame_rate = r(0)/2 + r(-1)/2
368 // (I.e. fr/s average this and the previous calculation.)
369 *framerate = (incoming_frame_rate_ + static_cast<unsigned int>(rate)) / 2;
370 incoming_frame_rate_ = static_cast<unsigned int>(rate);
371
372 // Calculate bit rate
373 if (incoming_bit_count_ == 0) {
374 *bitrate = 0;
375 } else {
philipel9d3ab612015-12-21 04:12:39 -0800376 *bitrate =
377 10 * ((100 * incoming_bit_count_) / static_cast<unsigned int>(diff));
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000378 }
379 incoming_bit_rate_ = *bitrate;
380
381 // Reset count
382 incoming_frame_count_ = 0;
383 incoming_bit_count_ = 0;
384 time_last_incoming_frame_count_ = now;
385
386 } else {
387 // No frames since last call
stefan@webrtc.orga678a3b2013-01-21 07:42:11 +0000388 time_last_incoming_frame_count_ = clock_->TimeInMilliseconds();
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000389 *framerate = 0;
stefan@webrtc.org49806792013-04-30 22:05:07 +0000390 *bitrate = 0;
391 incoming_frame_rate_ = 0;
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000392 incoming_bit_rate_ = 0;
393 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000394}
395
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000396// Returns immediately or a |max_wait_time_ms| ms event hang waiting for a
397// complete frame, |max_wait_time_ms| decided by caller.
isheriff6b4b5f32016-06-08 00:24:21 -0700398VCMEncodedFrame* VCMJitterBuffer::NextCompleteFrame(uint32_t max_wait_time_ms) {
kthelgasonff046c72017-03-31 02:03:55 -0700399 crit_sect_.Enter();
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000400 if (!running_) {
kthelgasonff046c72017-03-31 02:03:55 -0700401 crit_sect_.Leave();
isheriff6b4b5f32016-06-08 00:24:21 -0700402 return nullptr;
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000403 }
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000404 CleanUpOldOrEmptyFrames();
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000405
agalusza@google.comd177c102013-08-08 01:12:33 +0000406 if (decodable_frames_.empty() ||
407 decodable_frames_.Front()->GetState() != kStateComplete) {
philipel9d3ab612015-12-21 04:12:39 -0800408 const int64_t end_wait_time_ms =
409 clock_->TimeInMilliseconds() + max_wait_time_ms;
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000410 int64_t wait_time_ms = max_wait_time_ms;
411 while (wait_time_ms > 0) {
kthelgasonff046c72017-03-31 02:03:55 -0700412 crit_sect_.Leave();
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000413 const EventTypeWrapper ret =
philipel9d3ab612015-12-21 04:12:39 -0800414 frame_event_->Wait(static_cast<uint32_t>(wait_time_ms));
kthelgasonff046c72017-03-31 02:03:55 -0700415 crit_sect_.Enter();
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000416 if (ret == kEventSignaled) {
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000417 // Are we shutting down the jitter buffer?
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000418 if (!running_) {
kthelgasonff046c72017-03-31 02:03:55 -0700419 crit_sect_.Leave();
isheriff6b4b5f32016-06-08 00:24:21 -0700420 return nullptr;
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000421 }
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +0000422 // Finding oldest frame ready for decoder.
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000423 CleanUpOldOrEmptyFrames();
agalusza@google.comd177c102013-08-08 01:12:33 +0000424 if (decodable_frames_.empty() ||
425 decodable_frames_.Front()->GetState() != kStateComplete) {
stefan@webrtc.orga678a3b2013-01-21 07:42:11 +0000426 wait_time_ms = end_wait_time_ms - clock_->TimeInMilliseconds();
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000427 } else {
428 break;
429 }
430 } else {
mikhal@webrtc.org9c7685f2013-05-07 16:07:52 +0000431 break;
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000432 }
433 }
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000434 }
agalusza@google.comd177c102013-08-08 01:12:33 +0000435 if (decodable_frames_.empty() ||
436 decodable_frames_.Front()->GetState() != kStateComplete) {
kthelgasonff046c72017-03-31 02:03:55 -0700437 crit_sect_.Leave();
isheriff6b4b5f32016-06-08 00:24:21 -0700438 return nullptr;
mikhal@webrtc.orgdc3cd212013-04-25 20:27:04 +0000439 }
isheriff6b4b5f32016-06-08 00:24:21 -0700440 VCMEncodedFrame* encoded_frame = decodable_frames_.Front();
kthelgasonff046c72017-03-31 02:03:55 -0700441 crit_sect_.Leave();
isheriff6b4b5f32016-06-08 00:24:21 -0700442 return encoded_frame;
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000443}
444
mikhal@webrtc.org759b0412013-05-07 16:36:00 +0000445VCMEncodedFrame* VCMJitterBuffer::ExtractAndSetDecode(uint32_t timestamp) {
kthelgasonff046c72017-03-31 02:03:55 -0700446 rtc::CritScope cs(&crit_sect_);
mikhal@webrtc.org759b0412013-05-07 16:36:00 +0000447 if (!running_) {
448 return NULL;
449 }
450 // Extract the frame with the desired timestamp.
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000451 VCMFrameBuffer* frame = decodable_frames_.PopFrame(timestamp);
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +0000452 bool continuous = true;
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000453 if (!frame) {
454 frame = incomplete_frames_.PopFrame(timestamp);
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +0000455 if (frame)
456 continuous = last_decoded_state_.ContinuousFrame(frame);
457 else
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000458 return NULL;
mikhal@webrtc.org759b0412013-05-07 16:36:00 +0000459 }
mikhal@webrtc.orgcb20a5b2013-05-15 17:10:44 +0000460 // Frame pulled out from jitter buffer, update the jitter estimate.
mikhal@webrtc.org759b0412013-05-07 16:36:00 +0000461 const bool retransmitted = (frame->GetNackCount() > 0);
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000462 if (retransmitted) {
philipelae284082016-05-09 12:14:29 +0200463 if (WaitForRetransmissions())
464 jitter_estimate_.FrameNacked();
Niels Möllerf0eee002018-11-28 16:31:29 +0100465 } else if (frame->size() > 0) {
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000466 // Ignore retransmitted and empty frames.
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000467 if (waiting_for_completion_.latest_packet_time >= 0) {
468 UpdateJitterEstimate(waiting_for_completion_, true);
469 }
mikhal@webrtc.orgcb20a5b2013-05-15 17:10:44 +0000470 if (frame->GetState() == kStateComplete) {
471 UpdateJitterEstimate(*frame, false);
472 } else {
473 // Wait for this one to get complete.
Niels Möllerf0eee002018-11-28 16:31:29 +0100474 waiting_for_completion_.frame_size = frame->size();
philipel9d3ab612015-12-21 04:12:39 -0800475 waiting_for_completion_.latest_packet_time = frame->LatestPacketTimeMs();
Niels Möller23775882018-08-16 10:24:12 +0200476 waiting_for_completion_.timestamp = frame->Timestamp();
mikhal@webrtc.orgcb20a5b2013-05-15 17:10:44 +0000477 }
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000478 }
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000479
480 // The state must be changed to decoding before cleaning up zero sized
481 // frames to avoid empty frames being cleaned up and then given to the
mikhal@webrtc.orgcb20a5b2013-05-15 17:10:44 +0000482 // decoder. Propagates the missing_frame bit.
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +0000483 frame->PrepareForDecode(continuous);
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000484
mikhal@webrtc.orgcb20a5b2013-05-15 17:10:44 +0000485 // We have a frame - update the last decoded state and nack list.
mikhal@webrtc.org759b0412013-05-07 16:36:00 +0000486 last_decoded_state_.SetState(frame);
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000487 DropPacketsFromNackList(last_decoded_state_.sequence_num());
agalusza@google.comd818dcb2013-07-29 21:48:11 +0000488
489 if ((*frame).IsSessionComplete())
490 UpdateAveragePacketsPerFrame(frame->NumPackets());
491
mikhal@webrtc.org759b0412013-05-07 16:36:00 +0000492 return frame;
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000493}
494
495// Release frame when done with decoding. Should never be used to release
496// frames from within the jitter buffer.
497void VCMJitterBuffer::ReleaseFrame(VCMEncodedFrame* frame) {
sprang22691e02016-07-13 10:57:07 -0700498 RTC_CHECK(frame != nullptr);
kthelgasonff046c72017-03-31 02:03:55 -0700499 rtc::CritScope cs(&crit_sect_);
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000500 VCMFrameBuffer* frame_buffer = static_cast<VCMFrameBuffer*>(frame);
sprang22691e02016-07-13 10:57:07 -0700501 RecycleFrameBuffer(frame_buffer);
stefan@webrtc.org791eec72011-10-11 07:53:43 +0000502}
503
niklase@google.com470e71d2011-07-07 08:21:25 +0000504// Gets frame to use for this timestamp. If no match, get empty frame.
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000505VCMFrameBufferEnum VCMJitterBuffer::GetFrame(const VCMPacket& packet,
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000506 VCMFrameBuffer** frame,
507 FrameList** frame_list) {
508 *frame = incomplete_frames_.PopFrame(packet.timestamp);
509 if (*frame != NULL) {
510 *frame_list = &incomplete_frames_;
511 return kNoError;
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000512 }
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000513 *frame = decodable_frames_.PopFrame(packet.timestamp);
514 if (*frame != NULL) {
515 *frame_list = &decodable_frames_;
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000516 return kNoError;
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000517 }
mikhal@webrtc.org3c5a9242013-09-03 20:45:36 +0000518
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000519 *frame_list = NULL;
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000520 // No match, return empty frame.
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000521 *frame = GetEmptyFrame();
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000522 if (*frame == NULL) {
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +0000523 // No free frame! Try to reclaim some...
Mirko Bonadei675513b2017-11-09 11:09:25 +0100524 RTC_LOG(LS_WARNING) << "Unable to get empty frame; Recycling.";
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +0000525 bool found_key_frame = RecycleFramesUntilKeyFrame();
526 *frame = GetEmptyFrame();
sprang22691e02016-07-13 10:57:07 -0700527 RTC_CHECK(*frame);
stefan@webrtc.org34c5da62014-04-11 14:08:35 +0000528 if (!found_key_frame) {
sprang22691e02016-07-13 10:57:07 -0700529 RecycleFrameBuffer(*frame);
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000530 return kFlushIndicator;
stefan@webrtc.org34c5da62014-04-11 14:08:35 +0000531 }
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000532 }
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +0000533 (*frame)->Reset();
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000534 return kNoError;
niklase@google.com470e71d2011-07-07 08:21:25 +0000535}
536
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000537int64_t VCMJitterBuffer::LastPacketTime(const VCMEncodedFrame* frame,
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000538 bool* retransmitted) const {
539 assert(retransmitted);
kthelgasonff046c72017-03-31 02:03:55 -0700540 rtc::CritScope cs(&crit_sect_);
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000541 const VCMFrameBuffer* frame_buffer =
542 static_cast<const VCMFrameBuffer*>(frame);
543 *retransmitted = (frame_buffer->GetNackCount() > 0);
544 return frame_buffer->LatestPacketTimeMs();
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000545}
546
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000547VCMFrameBufferEnum VCMJitterBuffer::InsertPacket(const VCMPacket& packet,
548 bool* retransmitted) {
kthelgasonff046c72017-03-31 02:03:55 -0700549 rtc::CritScope cs(&crit_sect_);
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +0000550
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000551 ++num_packets_;
552 // Does this packet belong to an old frame?
553 if (last_decoded_state_.IsOldPacket(&packet)) {
554 // Account only for media packets.
555 if (packet.sizeBytes > 0) {
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000556 num_consecutive_old_packets_++;
557 }
558 // Update last decoded sequence number if the packet arrived late and
559 // belongs to a frame with a timestamp equal to the last decoded
560 // timestamp.
561 last_decoded_state_.UpdateOldPacket(&packet);
562 DropPacketsFromNackList(last_decoded_state_.sequence_num());
563
Noah Richardse4cb4e92015-05-22 14:03:00 -0700564 // Also see if this old packet made more incomplete frames continuous.
565 FindAndInsertContinuousFramesWithState(last_decoded_state_);
566
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000567 if (num_consecutive_old_packets_ > kMaxConsecutiveOldPackets) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100568 RTC_LOG(LS_WARNING)
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000569 << num_consecutive_old_packets_
570 << " consecutive old packets received. Flushing the jitter buffer.";
571 Flush();
572 return kFlushIndicator;
573 }
574 return kOldPacket;
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000575 }
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000576
asapersson9a4cd872015-10-23 00:27:14 -0700577 num_consecutive_old_packets_ = 0;
578
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000579 VCMFrameBuffer* frame;
580 FrameList* frame_list;
581 const VCMFrameBufferEnum error = GetFrame(packet, &frame, &frame_list);
582 if (error != kNoError)
583 return error;
584
mikhal@webrtc.org3c5a9242013-09-03 20:45:36 +0000585 int64_t now_ms = clock_->TimeInMilliseconds();
586 // We are keeping track of the first and latest seq numbers, and
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000587 // the number of wraps to be able to calculate how many packets we expect.
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000588 if (first_packet_since_reset_) {
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000589 // Now it's time to start estimating jitter
590 // reset the delay estimate.
mikhal@webrtc.org3c5a9242013-09-03 20:45:36 +0000591 inter_frame_delay_.Reset(now_ms);
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000592 }
593
594 // Empty packets may bias the jitter estimate (lacking size component),
595 // therefore don't let empty packet trigger the following updates:
Niels Möller8f7ce222019-03-21 15:43:58 +0100596 if (packet.frameType != VideoFrameType::kEmptyFrame) {
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000597 if (waiting_for_completion_.timestamp == packet.timestamp) {
598 // This can get bad if we have a lot of duplicate packets,
599 // we will then count some packet multiple times.
600 waiting_for_completion_.frame_size += packet.sizeBytes;
601 waiting_for_completion_.latest_packet_time = now_ms;
602 } else if (waiting_for_completion_.latest_packet_time >= 0 &&
603 waiting_for_completion_.latest_packet_time + 2000 <= now_ms) {
604 // A packet should never be more than two seconds late
605 UpdateJitterEstimate(waiting_for_completion_, true);
606 waiting_for_completion_.latest_packet_time = -1;
607 waiting_for_completion_.frame_size = 0;
608 waiting_for_completion_.timestamp = 0;
609 }
610 }
611
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000612 VCMFrameBufferStateEnum previous_state = frame->GetState();
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000613 // Insert packet.
agalusza@google.comd818dcb2013-07-29 21:48:11 +0000614 FrameData frame_data;
615 frame_data.rtt_ms = rtt_ms_;
616 frame_data.rolling_average_packets_per_frame = average_packets_per_frame_;
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000617 VCMFrameBufferEnum buffer_state =
Niels Möller375b3462019-01-10 15:35:56 +0100618 frame->InsertPacket(packet, now_ms, frame_data);
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000619
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000620 if (buffer_state > 0) {
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000621 incoming_bit_count_ += packet.sizeBytes << 3;
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000622 if (first_packet_since_reset_) {
623 latest_received_sequence_number_ = packet.seqNum;
624 first_packet_since_reset_ = false;
625 } else {
626 if (IsPacketRetransmitted(packet)) {
627 frame->IncrementNackCount();
628 }
pbos@webrtc.orgebb467f2014-05-19 15:28:02 +0000629 if (!UpdateNackList(packet.seqNum) &&
Niels Möller8f7ce222019-03-21 15:43:58 +0100630 packet.frameType != VideoFrameType::kVideoFrameKey) {
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000631 buffer_state = kFlushIndicator;
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000632 }
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000633
philipel9d3ab612015-12-21 04:12:39 -0800634 latest_received_sequence_number_ =
635 LatestSequenceNumber(latest_received_sequence_number_, packet.seqNum);
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000636 }
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000637 }
mikhal@webrtc.org3c5a9242013-09-03 20:45:36 +0000638
639 // Is the frame already in the decodable list?
mikhal@webrtc.org3c5a9242013-09-03 20:45:36 +0000640 bool continuous = IsContinuous(*frame);
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000641 switch (buffer_state) {
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000642 case kGeneralError:
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000643 case kTimeStampError:
644 case kSizeError: {
sprang22691e02016-07-13 10:57:07 -0700645 RecycleFrameBuffer(frame);
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000646 break;
647 }
648 case kCompleteSession: {
Niels Möller375b3462019-01-10 15:35:56 +0100649 if (previous_state != kStateComplete) {
mikhal@webrtc.org3c5a9242013-09-03 20:45:36 +0000650 CountFrame(*frame);
mikhal@webrtc.org3c5a9242013-09-03 20:45:36 +0000651 if (continuous) {
652 // Signal that we have a complete session.
653 frame_event_->Set();
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000654 }
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000655 }
Niels Möller375b3462019-01-10 15:35:56 +0100656
mikhal@webrtc.org3c5a9242013-09-03 20:45:36 +0000657 *retransmitted = (frame->GetNackCount() > 0);
mikhal@webrtc.org3c5a9242013-09-03 20:45:36 +0000658 if (continuous) {
mikhal@webrtc.org3c5a9242013-09-03 20:45:36 +0000659 decodable_frames_.InsertFrame(frame);
660 FindAndInsertContinuousFrames(*frame);
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000661 } else {
mikhal@webrtc.org3c5a9242013-09-03 20:45:36 +0000662 incomplete_frames_.InsertFrame(frame);
jbauchdb81ffd2015-11-23 03:59:02 -0800663 // If NACKs are enabled, keyframes are triggered by |GetNackList|.
Yves Gerey665174f2018-06-19 15:03:05 +0200664 if (nack_mode_ == kNoNack && NonContinuousOrIncompleteDuration() >
665 90 * kMaxDiscontinuousFramesTime) {
jbauchdb81ffd2015-11-23 03:59:02 -0800666 return kFlushIndicator;
667 }
mikhal@webrtc.org3c5a9242013-09-03 20:45:36 +0000668 }
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000669 break;
670 }
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000671 case kIncomplete: {
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000672 if (frame->GetState() == kStateEmpty &&
673 last_decoded_state_.UpdateEmptyFrame(frame)) {
sprang22691e02016-07-13 10:57:07 -0700674 RecycleFrameBuffer(frame);
mikhal@webrtc.org3c5a9242013-09-03 20:45:36 +0000675 return kNoError;
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000676 } else {
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000677 incomplete_frames_.InsertFrame(frame);
jbauchdb81ffd2015-11-23 03:59:02 -0800678 // If NACKs are enabled, keyframes are triggered by |GetNackList|.
Yves Gerey665174f2018-06-19 15:03:05 +0200679 if (nack_mode_ == kNoNack && NonContinuousOrIncompleteDuration() >
680 90 * kMaxDiscontinuousFramesTime) {
jbauchdb81ffd2015-11-23 03:59:02 -0800681 return kFlushIndicator;
682 }
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000683 }
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000684 break;
685 }
686 case kNoError:
mikhal@webrtc.orgf31a47a2013-08-26 17:10:11 +0000687 case kOutOfBoundsPacket:
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000688 case kDuplicatePacket: {
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000689 // Put back the frame where it came from.
690 if (frame_list != NULL) {
691 frame_list->InsertFrame(frame);
692 } else {
sprang22691e02016-07-13 10:57:07 -0700693 RecycleFrameBuffer(frame);
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000694 }
asapersson@webrtc.org96dc6852014-11-03 14:40:38 +0000695 ++num_duplicated_packets_;
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000696 break;
697 }
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000698 case kFlushIndicator:
sprang22691e02016-07-13 10:57:07 -0700699 RecycleFrameBuffer(frame);
mikhal@webrtc.org3c5a9242013-09-03 20:45:36 +0000700 return kFlushIndicator;
philipel9d3ab612015-12-21 04:12:39 -0800701 default:
702 assert(false);
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000703 }
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000704 return buffer_state;
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000705}
706
philipel9d3ab612015-12-21 04:12:39 -0800707bool VCMJitterBuffer::IsContinuousInState(
708 const VCMFrameBuffer& frame,
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000709 const VCMDecodingState& decoding_state) const {
Niels Möller375b3462019-01-10 15:35:56 +0100710 // Is this frame complete and continuous?
711 return (frame.GetState() == kStateComplete) &&
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000712 decoding_state.ContinuousFrame(&frame);
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000713}
714
715bool VCMJitterBuffer::IsContinuous(const VCMFrameBuffer& frame) const {
716 if (IsContinuousInState(frame, last_decoded_state_)) {
717 return true;
718 }
719 VCMDecodingState decoding_state;
720 decoding_state.CopyFrom(last_decoded_state_);
721 for (FrameList::const_iterator it = decodable_frames_.begin();
philipel9d3ab612015-12-21 04:12:39 -0800722 it != decodable_frames_.end(); ++it) {
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +0000723 VCMFrameBuffer* decodable_frame = it->second;
Niels Möller23775882018-08-16 10:24:12 +0200724 if (IsNewerTimestamp(decodable_frame->Timestamp(), frame.Timestamp())) {
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000725 break;
726 }
727 decoding_state.SetState(decodable_frame);
728 if (IsContinuousInState(frame, decoding_state)) {
729 return true;
730 }
731 }
732 return false;
733}
734
735void VCMJitterBuffer::FindAndInsertContinuousFrames(
736 const VCMFrameBuffer& new_frame) {
737 VCMDecodingState decoding_state;
738 decoding_state.CopyFrom(last_decoded_state_);
739 decoding_state.SetState(&new_frame);
Noah Richardse4cb4e92015-05-22 14:03:00 -0700740 FindAndInsertContinuousFramesWithState(decoding_state);
741}
742
743void VCMJitterBuffer::FindAndInsertContinuousFramesWithState(
744 const VCMDecodingState& original_decoded_state) {
745 // Copy original_decoded_state so we can move the state forward with each
746 // decodable frame we find.
747 VCMDecodingState decoding_state;
748 decoding_state.CopyFrom(original_decoded_state);
749
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000750 // When temporal layers are available, we search for a complete or decodable
751 // frame until we hit one of the following:
752 // 1. Continuous base or sync layer.
753 // 2. The end of the list was reached.
754 for (FrameList::iterator it = incomplete_frames_.begin();
philipel9d3ab612015-12-21 04:12:39 -0800755 it != incomplete_frames_.end();) {
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +0000756 VCMFrameBuffer* frame = it->second;
Noah Richardse4cb4e92015-05-22 14:03:00 -0700757 if (IsNewerTimestamp(original_decoded_state.time_stamp(),
Niels Möller23775882018-08-16 10:24:12 +0200758 frame->Timestamp())) {
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000759 ++it;
760 continue;
761 }
762 if (IsContinuousInState(*frame, decoding_state)) {
763 decodable_frames_.InsertFrame(frame);
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +0000764 incomplete_frames_.erase(it++);
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000765 decoding_state.SetState(frame);
766 } else if (frame->TemporalId() <= 0) {
767 break;
768 } else {
769 ++it;
770 }
771 }
772}
773
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000774uint32_t VCMJitterBuffer::EstimatedJitterMs() {
kthelgasonff046c72017-03-31 02:03:55 -0700775 rtc::CritScope cs(&crit_sect_);
mikhal@webrtc.org119c67d2013-01-31 17:18:02 +0000776 // Compute RTT multiplier for estimation.
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000777 // low_rtt_nackThresholdMs_ == -1 means no FEC.
778 double rtt_mult = 1.0f;
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000779 if (low_rtt_nack_threshold_ms_ >= 0 &&
pkasting@chromium.org16825b12015-01-12 21:51:21 +0000780 rtt_ms_ >= low_rtt_nack_threshold_ms_) {
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000781 // For RTTs above low_rtt_nack_threshold_ms_ we don't apply extra delay
782 // when waiting for retransmissions.
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000783 rtt_mult = 0.0f;
784 }
mikhal@webrtc.org119c67d2013-01-31 17:18:02 +0000785 return jitter_estimate_.GetJitterEstimate(rtt_mult);
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000786}
787
pkasting@chromium.org16825b12015-01-12 21:51:21 +0000788void VCMJitterBuffer::UpdateRtt(int64_t rtt_ms) {
kthelgasonff046c72017-03-31 02:03:55 -0700789 rtc::CritScope cs(&crit_sect_);
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000790 rtt_ms_ = rtt_ms;
791 jitter_estimate_.UpdateRtt(rtt_ms);
philipelae284082016-05-09 12:14:29 +0200792 if (!WaitForRetransmissions())
793 jitter_estimate_.ResetNackCount();
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000794}
795
796void VCMJitterBuffer::SetNackMode(VCMNackMode mode,
pkasting@chromium.org16825b12015-01-12 21:51:21 +0000797 int64_t low_rtt_nack_threshold_ms,
798 int64_t high_rtt_nack_threshold_ms) {
kthelgasonff046c72017-03-31 02:03:55 -0700799 rtc::CritScope cs(&crit_sect_);
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000800 nack_mode_ = mode;
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000801 if (mode == kNoNack) {
802 missing_sequence_numbers_.clear();
803 }
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000804 assert(low_rtt_nack_threshold_ms >= -1 && high_rtt_nack_threshold_ms >= -1);
805 assert(high_rtt_nack_threshold_ms == -1 ||
806 low_rtt_nack_threshold_ms <= high_rtt_nack_threshold_ms);
807 assert(low_rtt_nack_threshold_ms > -1 || high_rtt_nack_threshold_ms == -1);
808 low_rtt_nack_threshold_ms_ = low_rtt_nack_threshold_ms;
809 high_rtt_nack_threshold_ms_ = high_rtt_nack_threshold_ms;
mflodman@webrtc.org7c894b72012-11-26 12:40:15 +0000810 // Don't set a high start rtt if high_rtt_nack_threshold_ms_ is used, to not
Wan-Teh Changf2912872015-06-05 13:16:45 -0700811 // disable NACK in |kNack| mode.
mflodman@webrtc.org7c894b72012-11-26 12:40:15 +0000812 if (rtt_ms_ == kDefaultRtt && high_rtt_nack_threshold_ms_ != -1) {
813 rtt_ms_ = 0;
814 }
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000815 if (!WaitForRetransmissions()) {
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000816 jitter_estimate_.ResetNackCount();
817 }
818}
819
stefan@webrtc.orgbecf9c82013-02-01 15:09:57 +0000820void VCMJitterBuffer::SetNackSettings(size_t max_nack_list_size,
stefan@webrtc.orgef144882013-05-07 19:16:33 +0000821 int max_packet_age_to_nack,
822 int max_incomplete_time_ms) {
kthelgasonff046c72017-03-31 02:03:55 -0700823 rtc::CritScope cs(&crit_sect_);
stefan@webrtc.orgbecf9c82013-02-01 15:09:57 +0000824 assert(max_packet_age_to_nack >= 0);
stefan@webrtc.orgef144882013-05-07 19:16:33 +0000825 assert(max_incomplete_time_ms_ >= 0);
stefan@webrtc.orgbecf9c82013-02-01 15:09:57 +0000826 max_nack_list_size_ = max_nack_list_size;
827 max_packet_age_to_nack_ = max_packet_age_to_nack;
stefan@webrtc.orgef144882013-05-07 19:16:33 +0000828 max_incomplete_time_ms_ = max_incomplete_time_ms;
stefan@webrtc.orgbecf9c82013-02-01 15:09:57 +0000829}
830
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000831VCMNackMode VCMJitterBuffer::nack_mode() const {
kthelgasonff046c72017-03-31 02:03:55 -0700832 rtc::CritScope cs(&crit_sect_);
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000833 return nack_mode_;
834}
835
stefan@webrtc.orgef144882013-05-07 19:16:33 +0000836int VCMJitterBuffer::NonContinuousOrIncompleteDuration() {
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000837 if (incomplete_frames_.empty()) {
stefan@webrtc.orgef144882013-05-07 19:16:33 +0000838 return 0;
839 }
Niels Möller23775882018-08-16 10:24:12 +0200840 uint32_t start_timestamp = incomplete_frames_.Front()->Timestamp();
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000841 if (!decodable_frames_.empty()) {
Niels Möller23775882018-08-16 10:24:12 +0200842 start_timestamp = decodable_frames_.Back()->Timestamp();
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000843 }
Niels Möller23775882018-08-16 10:24:12 +0200844 return incomplete_frames_.Back()->Timestamp() - start_timestamp;
stefan@webrtc.orgef144882013-05-07 19:16:33 +0000845}
846
847uint16_t VCMJitterBuffer::EstimatedLowSequenceNumber(
848 const VCMFrameBuffer& frame) const {
849 assert(frame.GetLowSeqNum() >= 0);
850 if (frame.HaveFirstPacket())
851 return frame.GetLowSeqNum();
hclam@chromium.orgfe307e12013-05-16 21:19:59 +0000852
853 // This estimate is not accurate if more than one packet with lower sequence
854 // number is lost.
stefan@webrtc.orgef144882013-05-07 19:16:33 +0000855 return frame.GetLowSeqNum() - 1;
856}
857
Wan-Teh Changb1825a42015-06-03 15:03:35 -0700858std::vector<uint16_t> VCMJitterBuffer::GetNackList(bool* request_key_frame) {
kthelgasonff046c72017-03-31 02:03:55 -0700859 rtc::CritScope cs(&crit_sect_);
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000860 *request_key_frame = false;
861 if (nack_mode_ == kNoNack) {
Wan-Teh Changb1825a42015-06-03 15:03:35 -0700862 return std::vector<uint16_t>();
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000863 }
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000864 if (last_decoded_state_.in_initial_state()) {
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000865 VCMFrameBuffer* next_frame = NextFrame();
Niels Möller8f7ce222019-03-21 15:43:58 +0100866 const bool first_frame_is_key =
867 next_frame &&
868 next_frame->FrameType() == VideoFrameType::kVideoFrameKey &&
869 next_frame->HaveFirstPacket();
stefan@webrtc.org885cd132013-04-16 09:38:26 +0000870 if (!first_frame_is_key) {
philipel9d3ab612015-12-21 04:12:39 -0800871 bool have_non_empty_frame =
872 decodable_frames_.end() != find_if(decodable_frames_.begin(),
873 decodable_frames_.end(),
874 HasNonEmptyState);
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000875 if (!have_non_empty_frame) {
philipel9d3ab612015-12-21 04:12:39 -0800876 have_non_empty_frame =
877 incomplete_frames_.end() != find_if(incomplete_frames_.begin(),
878 incomplete_frames_.end(),
879 HasNonEmptyState);
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000880 }
stefan@webrtc.org885cd132013-04-16 09:38:26 +0000881 bool found_key_frame = RecycleFramesUntilKeyFrame();
882 if (!found_key_frame) {
883 *request_key_frame = have_non_empty_frame;
Wan-Teh Changb1825a42015-06-03 15:03:35 -0700884 return std::vector<uint16_t>();
stefan@webrtc.org885cd132013-04-16 09:38:26 +0000885 }
886 }
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000887 }
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000888 if (TooLargeNackList()) {
889 *request_key_frame = !HandleTooLargeNackList();
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000890 }
stefan@webrtc.orgef144882013-05-07 19:16:33 +0000891 if (max_incomplete_time_ms_ > 0) {
892 int non_continuous_incomplete_duration =
893 NonContinuousOrIncompleteDuration();
894 if (non_continuous_incomplete_duration > 90 * max_incomplete_time_ms_) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100895 RTC_LOG_F(LS_WARNING) << "Too long non-decodable duration: "
896 << non_continuous_incomplete_duration << " > "
897 << 90 * max_incomplete_time_ms_;
philipel9d3ab612015-12-21 04:12:39 -0800898 FrameList::reverse_iterator rit = find_if(
899 incomplete_frames_.rbegin(), incomplete_frames_.rend(), IsKeyFrame);
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000900 if (rit == incomplete_frames_.rend()) {
stefan@webrtc.orgef144882013-05-07 19:16:33 +0000901 // Request a key frame if we don't have one already.
902 *request_key_frame = true;
Wan-Teh Changb1825a42015-06-03 15:03:35 -0700903 return std::vector<uint16_t>();
stefan@webrtc.orgef144882013-05-07 19:16:33 +0000904 } else {
905 // Skip to the last key frame. If it's incomplete we will start
906 // NACKing it.
hclam@chromium.orgfe307e12013-05-16 21:19:59 +0000907 // Note that the estimated low sequence number is correct for VP8
908 // streams because only the first packet of a key frame is marked.
stefan@webrtc.orgef144882013-05-07 19:16:33 +0000909 last_decoded_state_.Reset();
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +0000910 DropPacketsFromNackList(EstimatedLowSequenceNumber(*rit->second));
stefan@webrtc.orgef144882013-05-07 19:16:33 +0000911 }
912 }
913 }
Wan-Teh Changb1825a42015-06-03 15:03:35 -0700914 std::vector<uint16_t> nack_list(missing_sequence_numbers_.begin(),
915 missing_sequence_numbers_.end());
916 return nack_list;
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000917}
918
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000919VCMFrameBuffer* VCMJitterBuffer::NextFrame() const {
920 if (!decodable_frames_.empty())
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +0000921 return decodable_frames_.Front();
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000922 if (!incomplete_frames_.empty())
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +0000923 return incomplete_frames_.Front();
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000924 return NULL;
925}
926
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000927bool VCMJitterBuffer::UpdateNackList(uint16_t sequence_number) {
928 if (nack_mode_ == kNoNack) {
929 return true;
930 }
stefan@webrtc.org4d2f5de2013-04-09 18:24:41 +0000931 // Make sure we don't add packets which are already too old to be decoded.
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000932 if (!last_decoded_state_.in_initial_state()) {
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000933 latest_received_sequence_number_ = LatestSequenceNumber(
philipel9d3ab612015-12-21 04:12:39 -0800934 latest_received_sequence_number_, last_decoded_state_.sequence_num());
stefan@webrtc.org4d2f5de2013-04-09 18:24:41 +0000935 }
stefan@webrtc.org7bc465b2013-04-11 17:48:02 +0000936 if (IsNewerSequenceNumber(sequence_number,
937 latest_received_sequence_number_)) {
stefan@webrtc.org4d2f5de2013-04-09 18:24:41 +0000938 // Push any missing sequence numbers to the NACK list.
939 for (uint16_t i = latest_received_sequence_number_ + 1;
stefan@webrtc.orga5dee332013-05-07 11:11:17 +0000940 IsNewerSequenceNumber(sequence_number, i); ++i) {
stefan@webrtc.org4d2f5de2013-04-09 18:24:41 +0000941 missing_sequence_numbers_.insert(missing_sequence_numbers_.end(), i);
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000942 }
stefan@webrtc.org4d2f5de2013-04-09 18:24:41 +0000943 if (TooLargeNackList() && !HandleTooLargeNackList()) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100944 RTC_LOG(LS_WARNING) << "Requesting key frame due to too large NACK list.";
stefan@webrtc.org4d2f5de2013-04-09 18:24:41 +0000945 return false;
946 }
947 if (MissingTooOldPacket(sequence_number) &&
948 !HandleTooOldPackets(sequence_number)) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100949 RTC_LOG(LS_WARNING)
950 << "Requesting key frame due to missing too old packets";
stefan@webrtc.org4d2f5de2013-04-09 18:24:41 +0000951 return false;
952 }
953 } else {
954 missing_sequence_numbers_.erase(sequence_number);
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000955 }
956 return true;
957}
958
959bool VCMJitterBuffer::TooLargeNackList() const {
960 return missing_sequence_numbers_.size() > max_nack_list_size_;
961}
962
963bool VCMJitterBuffer::HandleTooLargeNackList() {
964 // Recycle frames until the NACK list is small enough. It is likely cheaper to
965 // request a key frame than to retransmit this many missing packets.
Mirko Bonadei675513b2017-11-09 11:09:25 +0100966 RTC_LOG_F(LS_WARNING) << "NACK list has grown too large: "
967 << missing_sequence_numbers_.size() << " > "
968 << max_nack_list_size_;
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000969 bool key_frame_found = false;
stefan@webrtc.org4d2f5de2013-04-09 18:24:41 +0000970 while (TooLargeNackList()) {
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000971 key_frame_found = RecycleFramesUntilKeyFrame();
972 }
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000973 return key_frame_found;
974}
975
976bool VCMJitterBuffer::MissingTooOldPacket(
977 uint16_t latest_sequence_number) const {
978 if (missing_sequence_numbers_.empty()) {
979 return false;
980 }
philipel9d3ab612015-12-21 04:12:39 -0800981 const uint16_t age_of_oldest_missing_packet =
982 latest_sequence_number - *missing_sequence_numbers_.begin();
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000983 // Recycle frames if the NACK list contains too old sequence numbers as
984 // the packets may have already been dropped by the sender.
985 return age_of_oldest_missing_packet > max_packet_age_to_nack_;
986}
987
988bool VCMJitterBuffer::HandleTooOldPackets(uint16_t latest_sequence_number) {
989 bool key_frame_found = false;
philipel9d3ab612015-12-21 04:12:39 -0800990 const uint16_t age_of_oldest_missing_packet =
991 latest_sequence_number - *missing_sequence_numbers_.begin();
Mirko Bonadei675513b2017-11-09 11:09:25 +0100992 RTC_LOG_F(LS_WARNING) << "NACK list contains too old sequence numbers: "
993 << age_of_oldest_missing_packet << " > "
994 << max_packet_age_to_nack_;
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000995 while (MissingTooOldPacket(latest_sequence_number)) {
996 key_frame_found = RecycleFramesUntilKeyFrame();
997 }
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000998 return key_frame_found;
999}
1000
1001void VCMJitterBuffer::DropPacketsFromNackList(
1002 uint16_t last_decoded_sequence_number) {
1003 // Erase all sequence numbers from the NACK list which we won't need any
1004 // longer.
philipel9d3ab612015-12-21 04:12:39 -08001005 missing_sequence_numbers_.erase(
1006 missing_sequence_numbers_.begin(),
1007 missing_sequence_numbers_.upper_bound(last_decoded_sequence_number));
stefan@webrtc.orga64300a2013-03-04 15:24:40 +00001008}
1009
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001010VCMFrameBuffer* VCMJitterBuffer::GetEmptyFrame() {
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +00001011 if (free_frames_.empty()) {
1012 if (!TryToIncreaseJitterBufferSize()) {
1013 return NULL;
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001014 }
1015 }
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +00001016 VCMFrameBuffer* frame = free_frames_.front();
1017 free_frames_.pop_front();
1018 return frame;
1019}
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001020
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +00001021bool VCMJitterBuffer::TryToIncreaseJitterBufferSize() {
1022 if (max_number_of_frames_ >= kMaxNumberOfFrames)
1023 return false;
pbos@webrtc.org4f16c872014-11-24 09:06:48 +00001024 free_frames_.push_back(new VCMFrameBuffer());
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +00001025 ++max_number_of_frames_;
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +00001026 return true;
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001027}
1028
1029// Recycle oldest frames up to a key frame, used if jitter buffer is completely
1030// full.
1031bool VCMJitterBuffer::RecycleFramesUntilKeyFrame() {
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +00001032 // First release incomplete frames, and only release decodable frames if there
1033 // are no incomplete ones.
1034 FrameList::iterator key_frame_it;
1035 bool key_frame_found = false;
1036 int dropped_frames = 0;
1037 dropped_frames += incomplete_frames_.RecycleFramesUntilKeyFrame(
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +00001038 &key_frame_it, &free_frames_);
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +00001039 key_frame_found = key_frame_it != incomplete_frames_.end();
1040 if (dropped_frames == 0) {
1041 dropped_frames += decodable_frames_.RecycleFramesUntilKeyFrame(
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +00001042 &key_frame_it, &free_frames_);
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +00001043 key_frame_found = key_frame_it != decodable_frames_.end();
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001044 }
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +00001045 if (key_frame_found) {
Mirko Bonadei675513b2017-11-09 11:09:25 +01001046 RTC_LOG(LS_INFO) << "Found key frame while dropping frames.";
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +00001047 // Reset last decoded state to make sure the next frame decoded is a key
1048 // frame, and start NACKing from here.
1049 last_decoded_state_.Reset();
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +00001050 DropPacketsFromNackList(EstimatedLowSequenceNumber(*key_frame_it->second));
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +00001051 } else if (decodable_frames_.empty()) {
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +00001052 // All frames dropped. Reset the decoding state and clear missing sequence
1053 // numbers as we're starting fresh.
1054 last_decoded_state_.Reset();
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +00001055 missing_sequence_numbers_.clear();
edjee@google.com79b02892013-04-04 19:43:34 +00001056 }
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +00001057 return key_frame_found;
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001058}
1059
1060// Must be called under the critical section |crit_sect_|.
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +00001061void VCMJitterBuffer::CountFrame(const VCMFrameBuffer& frame) {
pbos@webrtc.org4f16c872014-11-24 09:06:48 +00001062 incoming_frame_count_++;
niklase@google.com470e71d2011-07-07 08:21:25 +00001063}
1064
agalusza@google.comd818dcb2013-07-29 21:48:11 +00001065void VCMJitterBuffer::UpdateAveragePacketsPerFrame(int current_number_packets) {
1066 if (frame_counter_ > kFastConvergeThreshold) {
philipel9d3ab612015-12-21 04:12:39 -08001067 average_packets_per_frame_ =
1068 average_packets_per_frame_ * (1 - kNormalConvergeMultiplier) +
1069 current_number_packets * kNormalConvergeMultiplier;
agalusza@google.comd818dcb2013-07-29 21:48:11 +00001070 } else if (frame_counter_ > 0) {
philipel9d3ab612015-12-21 04:12:39 -08001071 average_packets_per_frame_ =
1072 average_packets_per_frame_ * (1 - kFastConvergeMultiplier) +
1073 current_number_packets * kFastConvergeMultiplier;
agalusza@google.comd818dcb2013-07-29 21:48:11 +00001074 frame_counter_++;
1075 } else {
1076 average_packets_per_frame_ = current_number_packets;
1077 frame_counter_++;
1078 }
1079}
1080
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001081// Must be called under the critical section |crit_sect_|.
stefan@webrtc.orga64300a2013-03-04 15:24:40 +00001082void VCMJitterBuffer::CleanUpOldOrEmptyFrames() {
pbos@webrtc.org4f16c872014-11-24 09:06:48 +00001083 decodable_frames_.CleanUpOldOrEmptyFrames(&last_decoded_state_,
1084 &free_frames_);
1085 incomplete_frames_.CleanUpOldOrEmptyFrames(&last_decoded_state_,
1086 &free_frames_);
stefan@webrtc.orga64300a2013-03-04 15:24:40 +00001087 if (!last_decoded_state_.in_initial_state()) {
1088 DropPacketsFromNackList(last_decoded_state_.sequence_num());
1089 }
mikhal@webrtc.org832caca2011-12-13 21:15:05 +00001090}
1091
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001092// Must be called from within |crit_sect_|.
1093bool VCMJitterBuffer::IsPacketRetransmitted(const VCMPacket& packet) const {
stefan@webrtc.orga64300a2013-03-04 15:24:40 +00001094 return missing_sequence_numbers_.find(packet.seqNum) !=
philipel9d3ab612015-12-21 04:12:39 -08001095 missing_sequence_numbers_.end();
niklase@google.com470e71d2011-07-07 08:21:25 +00001096}
1097
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001098// Must be called under the critical section |crit_sect_|. Should never be
1099// called with retransmitted frames, they must be filtered out before this
1100// function is called.
1101void VCMJitterBuffer::UpdateJitterEstimate(const VCMJitterSample& sample,
1102 bool incomplete_frame) {
1103 if (sample.latest_packet_time == -1) {
1104 return;
1105 }
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001106 UpdateJitterEstimate(sample.latest_packet_time, sample.timestamp,
1107 sample.frame_size, incomplete_frame);
1108}
1109
1110// Must be called under the critical section crit_sect_. Should never be
1111// called with retransmitted frames, they must be filtered out before this
1112// function is called.
1113void VCMJitterBuffer::UpdateJitterEstimate(const VCMFrameBuffer& frame,
1114 bool incomplete_frame) {
1115 if (frame.LatestPacketTimeMs() == -1) {
1116 return;
1117 }
1118 // No retransmitted frames should be a part of the jitter
1119 // estimate.
Niels Möller23775882018-08-16 10:24:12 +02001120 UpdateJitterEstimate(frame.LatestPacketTimeMs(), frame.Timestamp(),
Niels Möllerf0eee002018-11-28 16:31:29 +01001121 frame.size(), incomplete_frame);
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001122}
1123
1124// Must be called under the critical section |crit_sect_|. Should never be
1125// called with retransmitted frames, they must be filtered out before this
1126// function is called.
philipel9d3ab612015-12-21 04:12:39 -08001127void VCMJitterBuffer::UpdateJitterEstimate(int64_t latest_packet_time_ms,
1128 uint32_t timestamp,
1129 unsigned int frame_size,
1130 bool incomplete_frame) {
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001131 if (latest_packet_time_ms == -1) {
1132 return;
1133 }
1134 int64_t frame_delay;
philipel9d3ab612015-12-21 04:12:39 -08001135 bool not_reordered = inter_frame_delay_.CalculateDelay(
1136 timestamp, &frame_delay, latest_packet_time_ms);
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001137 // Filter out frames which have been reordered in time by the network
1138 if (not_reordered) {
1139 // Update the jitter estimate with the new samples
1140 jitter_estimate_.UpdateEstimate(frame_delay, frame_size, incomplete_frame);
1141 }
1142}
1143
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001144bool VCMJitterBuffer::WaitForRetransmissions() {
1145 if (nack_mode_ == kNoNack) {
1146 // NACK disabled -> don't wait for retransmissions.
1147 return false;
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001148 }
stefan@webrtc.orga64300a2013-03-04 15:24:40 +00001149 // Evaluate if the RTT is higher than |high_rtt_nack_threshold_ms_|, and in
1150 // that case we don't wait for retransmissions.
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001151 if (high_rtt_nack_threshold_ms_ >= 0 &&
pkasting@chromium.org16825b12015-01-12 21:51:21 +00001152 rtt_ms_ >= high_rtt_nack_threshold_ms_) {
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001153 return false;
1154 }
1155 return true;
1156}
philipel83f831a2016-03-12 03:30:23 -08001157
sprang22691e02016-07-13 10:57:07 -07001158void VCMJitterBuffer::RecycleFrameBuffer(VCMFrameBuffer* frame) {
1159 frame->Reset();
1160 free_frames_.push_back(frame);
1161}
1162
stefan@webrtc.org932ab182011-11-29 11:33:31 +00001163} // namespace webrtc