blob: a381880891b09b217d6bfbfd064a924b414ee08b [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:25 +00001/*
stefan@webrtc.org29794612012-02-08 08:58:55 +00002 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
niklase@google.com470e71d2011-07-07 08:21:25 +00003 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
Henrik Kjellander2557b862015-11-18 22:00:21 +010010#include "webrtc/modules/video_coding/jitter_buffer.h"
niklase@google.com470e71d2011-07-07 08:21:25 +000011
pbos@webrtc.org12dc1a32013-08-05 16:22:53 +000012#include <assert.h>
13
stefan@webrtc.org29794612012-02-08 08:58:55 +000014#include <algorithm>
agalusza@google.comd818dcb2013-07-29 21:48:11 +000015#include <utility>
stefan@webrtc.org29794612012-02-08 08:58:55 +000016
asapersson9a4cd872015-10-23 00:27:14 -070017#include "webrtc/base/checks.h"
pbos854e84c2015-11-16 16:39:06 -080018#include "webrtc/base/logging.h"
tommie4f96502015-10-20 23:00:48 -070019#include "webrtc/base/trace_event.h"
Henrik Kjellanderff761fb2015-11-04 08:31:52 +010020#include "webrtc/modules/rtp_rtcp/include/rtp_rtcp_defines.h"
Henrik Kjellander2557b862015-11-18 22:00:21 +010021#include "webrtc/modules/video_coding/include/video_coding.h"
22#include "webrtc/modules/video_coding/frame_buffer.h"
23#include "webrtc/modules/video_coding/inter_frame_delay.h"
24#include "webrtc/modules/video_coding/internal_defines.h"
25#include "webrtc/modules/video_coding/jitter_buffer_common.h"
26#include "webrtc/modules/video_coding/jitter_estimator.h"
27#include "webrtc/modules/video_coding/packet.h"
Henrik Kjellander98f53512015-10-28 18:17:40 +010028#include "webrtc/system_wrappers/include/clock.h"
29#include "webrtc/system_wrappers/include/critical_section_wrapper.h"
30#include "webrtc/system_wrappers/include/event_wrapper.h"
Henrik Kjellander98f53512015-10-28 18:17:40 +010031#include "webrtc/system_wrappers/include/metrics.h"
niklase@google.com470e71d2011-07-07 08:21:25 +000032
niklase@google.com470e71d2011-07-07 08:21:25 +000033namespace webrtc {
34
asapersson9a4cd872015-10-23 00:27:14 -070035// Interval for updating SS data.
36static const uint32_t kSsCleanupIntervalSec = 60;
37
mflodman@webrtc.org7c894b72012-11-26 12:40:15 +000038// Use this rtt if no value has been reported.
pkasting@chromium.org16825b12015-01-12 21:51:21 +000039static const int64_t kDefaultRtt = 200;
mflodman@webrtc.org7c894b72012-11-26 12:40:15 +000040
jbauchdb81ffd2015-11-23 03:59:02 -080041// Request a keyframe if no continuous frame has been received for this
42// number of milliseconds and NACKs are disabled.
43static const int64_t kMaxDiscontinuousFramesTime = 1000;
44
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +000045typedef std::pair<uint32_t, VCMFrameBuffer*> FrameListPair;
niklase@google.com470e71d2011-07-07 08:21:25 +000046
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +000047bool IsKeyFrame(FrameListPair pair) {
48 return pair.second->FrameType() == kVideoFrameKey;
49}
stefan@webrtc.org29794612012-02-08 08:58:55 +000050
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +000051bool HasNonEmptyState(FrameListPair pair) {
52 return pair.second->GetState() != kStateEmpty;
stefan@webrtc.orga64300a2013-03-04 15:24:40 +000053}
54
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +000055void FrameList::InsertFrame(VCMFrameBuffer* frame) {
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +000056 insert(rbegin().base(), FrameListPair(frame->TimeStamp(), frame));
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +000057}
58
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +000059VCMFrameBuffer* FrameList::PopFrame(uint32_t timestamp) {
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +000060 FrameList::iterator it = find(timestamp);
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +000061 if (it == end())
62 return NULL;
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +000063 VCMFrameBuffer* frame = it->second;
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +000064 erase(it);
65 return frame;
66}
67
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +000068VCMFrameBuffer* FrameList::Front() const {
69 return begin()->second;
70}
71
72VCMFrameBuffer* FrameList::Back() const {
73 return rbegin()->second;
74}
75
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +000076int FrameList::RecycleFramesUntilKeyFrame(FrameList::iterator* key_frame_it,
77 UnorderedFrameList* free_frames) {
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +000078 int drop_count = 0;
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +000079 FrameList::iterator it = begin();
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +000080 while (!empty()) {
81 // Throw at least one frame.
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +000082 it->second->Reset();
83 free_frames->push_back(it->second);
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +000084 erase(it++);
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +000085 ++drop_count;
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +000086 if (it != end() && it->second->FrameType() == kVideoFrameKey) {
87 *key_frame_it = it;
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +000088 return drop_count;
89 }
90 }
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +000091 *key_frame_it = end();
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +000092 return drop_count;
93}
94
pbos@webrtc.org4f16c872014-11-24 09:06:48 +000095void FrameList::CleanUpOldOrEmptyFrames(VCMDecodingState* decoding_state,
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +000096 UnorderedFrameList* free_frames) {
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +000097 while (!empty()) {
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +000098 VCMFrameBuffer* oldest_frame = Front();
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +000099 bool remove_frame = false;
100 if (oldest_frame->GetState() == kStateEmpty && size() > 1) {
101 // This frame is empty, try to update the last decoded state and drop it
102 // if successful.
103 remove_frame = decoding_state->UpdateEmptyFrame(oldest_frame);
104 } else {
105 remove_frame = decoding_state->IsOldFrame(oldest_frame);
106 }
107 if (!remove_frame) {
108 break;
109 }
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +0000110 free_frames->push_back(oldest_frame);
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000111 TRACE_EVENT_INSTANT1("webrtc", "JB::OldOrEmptyFrameDropped", "timestamp",
112 oldest_frame->TimeStamp());
113 erase(begin());
114 }
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000115}
116
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +0000117void FrameList::Reset(UnorderedFrameList* free_frames) {
118 while (!empty()) {
119 begin()->second->Reset();
120 free_frames->push_back(begin()->second);
121 erase(begin());
122 }
123}
124
asapersson9a4cd872015-10-23 00:27:14 -0700125bool Vp9SsMap::Insert(const VCMPacket& packet) {
126 if (!packet.codecSpecificHeader.codecHeader.VP9.ss_data_available)
127 return false;
128
129 ss_map_[packet.timestamp] = packet.codecSpecificHeader.codecHeader.VP9.gof;
130 return true;
131}
132
133void Vp9SsMap::Reset() {
134 ss_map_.clear();
135}
136
137bool Vp9SsMap::Find(uint32_t timestamp, SsMap::iterator* it_out) {
138 bool found = false;
139 for (SsMap::iterator it = ss_map_.begin(); it != ss_map_.end(); ++it) {
140 if (it->first == timestamp || IsNewerTimestamp(timestamp, it->first)) {
141 *it_out = it;
142 found = true;
143 }
144 }
145 return found;
146}
147
148void Vp9SsMap::RemoveOld(uint32_t timestamp) {
149 if (!TimeForCleanup(timestamp))
150 return;
151
152 SsMap::iterator it;
153 if (!Find(timestamp, &it))
154 return;
155
156 ss_map_.erase(ss_map_.begin(), it);
157 AdvanceFront(timestamp);
158}
159
160bool Vp9SsMap::TimeForCleanup(uint32_t timestamp) const {
161 if (ss_map_.empty() || !IsNewerTimestamp(timestamp, ss_map_.begin()->first))
162 return false;
163
164 uint32_t diff = timestamp - ss_map_.begin()->first;
165 return diff / kVideoPayloadTypeFrequency >= kSsCleanupIntervalSec;
166}
167
168void Vp9SsMap::AdvanceFront(uint32_t timestamp) {
169 RTC_DCHECK(!ss_map_.empty());
170 GofInfoVP9 gof = ss_map_.begin()->second;
171 ss_map_.erase(ss_map_.begin());
172 ss_map_[timestamp] = gof;
173}
174
asaperssonc253a1c2015-11-06 00:12:01 -0800175// TODO(asapersson): Update according to updates in RTP payload profile.
asapersson9a4cd872015-10-23 00:27:14 -0700176bool Vp9SsMap::UpdatePacket(VCMPacket* packet) {
177 uint8_t gof_idx = packet->codecSpecificHeader.codecHeader.VP9.gof_idx;
178 if (gof_idx == kNoGofIdx)
179 return false; // No update needed.
180
181 SsMap::iterator it;
182 if (!Find(packet->timestamp, &it))
183 return false; // Corresponding SS not yet received.
184
185 if (gof_idx >= it->second.num_frames_in_gof)
186 return false; // Assume corresponding SS not yet received.
187
188 RTPVideoHeaderVP9* vp9 = &packet->codecSpecificHeader.codecHeader.VP9;
189 vp9->temporal_idx = it->second.temporal_idx[gof_idx];
190 vp9->temporal_up_switch = it->second.temporal_up_switch[gof_idx];
191
192 // TODO(asapersson): Set vp9.ref_picture_id[i] and add usage.
193 vp9->num_ref_pics = it->second.num_ref_pics[gof_idx];
asaperssonc253a1c2015-11-06 00:12:01 -0800194 for (uint8_t i = 0; i < it->second.num_ref_pics[gof_idx]; ++i) {
asapersson9a4cd872015-10-23 00:27:14 -0700195 vp9->pid_diff[i] = it->second.pid_diff[gof_idx][i];
196 }
197 return true;
198}
199
200void Vp9SsMap::UpdateFrames(FrameList* frames) {
201 for (const auto& frame_it : *frames) {
202 uint8_t gof_idx =
203 frame_it.second->CodecSpecific()->codecSpecific.VP9.gof_idx;
204 if (gof_idx == kNoGofIdx) {
205 continue;
206 }
207 SsMap::iterator ss_it;
208 if (Find(frame_it.second->TimeStamp(), &ss_it)) {
209 if (gof_idx >= ss_it->second.num_frames_in_gof) {
210 continue; // Assume corresponding SS not yet received.
211 }
212 frame_it.second->SetGofInfo(ss_it->second, gof_idx);
213 }
214 }
215}
216
Qiang Chend4cec152015-06-19 09:17:00 -0700217VCMJitterBuffer::VCMJitterBuffer(Clock* clock,
218 rtc::scoped_ptr<EventWrapper> event)
stefan@webrtc.org34c5da62014-04-11 14:08:35 +0000219 : clock_(clock),
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000220 running_(false),
221 crit_sect_(CriticalSectionWrapper::CreateCriticalSection()),
kwiberg0eb15ed2015-12-17 03:04:15 -0800222 frame_event_(std::move(event)),
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000223 max_number_of_frames_(kStartNumberOfFrames),
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +0000224 free_frames_(),
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000225 decodable_frames_(),
226 incomplete_frames_(),
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000227 last_decoded_state_(),
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000228 first_packet_since_reset_(true),
pbos@webrtc.org55707692014-12-19 15:45:03 +0000229 stats_callback_(NULL),
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000230 incoming_frame_rate_(0),
231 incoming_frame_count_(0),
232 time_last_incoming_frame_count_(0),
233 incoming_bit_count_(0),
234 incoming_bit_rate_(0),
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000235 num_consecutive_old_packets_(0),
asapersson@webrtc.org96dc6852014-11-03 14:40:38 +0000236 num_packets_(0),
237 num_duplicated_packets_(0),
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000238 num_discarded_packets_(0),
asapersson@webrtc.org83b52002014-11-28 10:17:13 +0000239 time_first_packet_ms_(0),
sprang@webrtc.org70e2d112014-09-24 14:06:56 +0000240 jitter_estimate_(clock),
stefan@webrtc.orga678a3b2013-01-21 07:42:11 +0000241 inter_frame_delay_(clock_->TimeInMilliseconds()),
mflodman@webrtc.org7c894b72012-11-26 12:40:15 +0000242 rtt_ms_(kDefaultRtt),
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000243 nack_mode_(kNoNack),
244 low_rtt_nack_threshold_ms_(-1),
245 high_rtt_nack_threshold_ms_(-1),
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000246 missing_sequence_numbers_(SequenceNumberLessThan()),
stefan@webrtc.orgbecf9c82013-02-01 15:09:57 +0000247 max_nack_list_size_(0),
248 max_packet_age_to_nack_(0),
stefan@webrtc.orgef144882013-05-07 19:16:33 +0000249 max_incomplete_time_ms_(0),
agalusza@google.coma7e360e2013-08-01 03:15:08 +0000250 decode_error_mode_(kNoErrors),
agalusza@google.comd818dcb2013-07-29 21:48:11 +0000251 average_packets_per_frame_(0.0f),
252 frame_counter_(0) {
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000253 for (int i = 0; i < kStartNumberOfFrames; i++)
254 free_frames_.push_back(new VCMFrameBuffer());
niklase@google.com470e71d2011-07-07 08:21:25 +0000255}
256
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000257VCMJitterBuffer::~VCMJitterBuffer() {
258 Stop();
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000259 for (UnorderedFrameList::iterator it = free_frames_.begin();
260 it != free_frames_.end(); ++it) {
261 delete *it;
262 }
263 for (FrameList::iterator it = incomplete_frames_.begin();
264 it != incomplete_frames_.end(); ++it) {
265 delete it->second;
266 }
267 for (FrameList::iterator it = decodable_frames_.begin();
268 it != decodable_frames_.end(); ++it) {
269 delete it->second;
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000270 }
271 delete crit_sect_;
niklase@google.com470e71d2011-07-07 08:21:25 +0000272}
273
asapersson@webrtc.org96dc6852014-11-03 14:40:38 +0000274void VCMJitterBuffer::UpdateHistograms() {
Ã…sa Perssona96f02b2015-04-24 08:52:11 +0200275 if (num_packets_ <= 0 || !running_) {
asapersson@webrtc.org83b52002014-11-28 10:17:13 +0000276 return;
277 }
278 int64_t elapsed_sec =
279 (clock_->TimeInMilliseconds() - time_first_packet_ms_) / 1000;
280 if (elapsed_sec < metrics::kMinRunTimeInSeconds) {
281 return;
282 }
283
asapersson53805322015-12-21 01:46:20 -0800284 RTC_HISTOGRAM_PERCENTAGE_SPARSE("WebRTC.Video.DiscardedPacketsInPercent",
285 num_discarded_packets_ * 100 / num_packets_);
286 RTC_HISTOGRAM_PERCENTAGE_SPARSE("WebRTC.Video.DuplicatedPacketsInPercent",
287 num_duplicated_packets_ * 100 / num_packets_);
asapersson@webrtc.org83b52002014-11-28 10:17:13 +0000288
pbos@webrtc.orgce4e9a32014-12-18 13:50:16 +0000289 int total_frames =
290 receive_statistics_.key_frames + receive_statistics_.delta_frames;
asapersson@webrtc.org83b52002014-11-28 10:17:13 +0000291 if (total_frames > 0) {
asapersson53805322015-12-21 01:46:20 -0800292 RTC_HISTOGRAM_COUNTS_SPARSE_100(
293 "WebRTC.Video.CompleteFramesReceivedPerSecond",
asapersson@webrtc.org83b52002014-11-28 10:17:13 +0000294 static_cast<int>((total_frames / elapsed_sec) + 0.5f));
asapersson53805322015-12-21 01:46:20 -0800295 RTC_HISTOGRAM_COUNTS_SPARSE_1000(
pbos@webrtc.orgce4e9a32014-12-18 13:50:16 +0000296 "WebRTC.Video.KeyFramesReceivedInPermille",
297 static_cast<int>(
298 (receive_statistics_.key_frames * 1000.0f / total_frames) + 0.5f));
asapersson@webrtc.org96dc6852014-11-03 14:40:38 +0000299 }
300}
301
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000302void VCMJitterBuffer::Start() {
303 CriticalSectionScoped cs(crit_sect_);
304 running_ = true;
305 incoming_frame_count_ = 0;
306 incoming_frame_rate_ = 0;
307 incoming_bit_count_ = 0;
308 incoming_bit_rate_ = 0;
stefan@webrtc.orga678a3b2013-01-21 07:42:11 +0000309 time_last_incoming_frame_count_ = clock_->TimeInMilliseconds();
pbos@webrtc.orgce4e9a32014-12-18 13:50:16 +0000310 receive_statistics_ = FrameCounts();
niklase@google.com470e71d2011-07-07 08:21:25 +0000311
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000312 num_consecutive_old_packets_ = 0;
asapersson@webrtc.org96dc6852014-11-03 14:40:38 +0000313 num_packets_ = 0;
314 num_duplicated_packets_ = 0;
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000315 num_discarded_packets_ = 0;
asapersson@webrtc.org83b52002014-11-28 10:17:13 +0000316 time_first_packet_ms_ = 0;
niklase@google.com470e71d2011-07-07 08:21:25 +0000317
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000318 // Start in a non-signaled state.
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000319 waiting_for_completion_.frame_size = 0;
320 waiting_for_completion_.timestamp = 0;
321 waiting_for_completion_.latest_packet_time = -1;
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000322 first_packet_since_reset_ = true;
mflodman@webrtc.org7c894b72012-11-26 12:40:15 +0000323 rtt_ms_ = kDefaultRtt;
mikhal@webrtc.org8392cd92013-04-25 21:30:50 +0000324 last_decoded_state_.Reset();
niklase@google.com470e71d2011-07-07 08:21:25 +0000325}
326
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000327void VCMJitterBuffer::Stop() {
328 crit_sect_->Enter();
asapersson@webrtc.org96dc6852014-11-03 14:40:38 +0000329 UpdateHistograms();
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000330 running_ = false;
331 last_decoded_state_.Reset();
asaperssona9455ab2015-07-31 06:10:09 -0700332
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000333 // Make sure all frames are free and reset.
334 for (FrameList::iterator it = decodable_frames_.begin();
335 it != decodable_frames_.end(); ++it) {
336 free_frames_.push_back(it->second);
337 }
338 for (FrameList::iterator it = incomplete_frames_.begin();
339 it != incomplete_frames_.end(); ++it) {
340 free_frames_.push_back(it->second);
341 }
342 for (UnorderedFrameList::iterator it = free_frames_.begin();
343 it != free_frames_.end(); ++it) {
344 (*it)->Reset();
345 }
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000346 decodable_frames_.clear();
347 incomplete_frames_.clear();
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000348 crit_sect_->Leave();
349 // Make sure we wake up any threads waiting on these events.
stefan@webrtc.org2baf5f52013-03-13 08:46:25 +0000350 frame_event_->Set();
niklase@google.com470e71d2011-07-07 08:21:25 +0000351}
352
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000353bool VCMJitterBuffer::Running() const {
354 CriticalSectionScoped cs(crit_sect_);
355 return running_;
niklase@google.com470e71d2011-07-07 08:21:25 +0000356}
357
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000358void VCMJitterBuffer::Flush() {
359 CriticalSectionScoped cs(crit_sect_);
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +0000360 decodable_frames_.Reset(&free_frames_);
361 incomplete_frames_.Reset(&free_frames_);
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000362 last_decoded_state_.Reset(); // TODO(mikhal): sync reset.
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000363 num_consecutive_old_packets_ = 0;
364 // Also reset the jitter and delay estimates
365 jitter_estimate_.Reset();
stefan@webrtc.orga678a3b2013-01-21 07:42:11 +0000366 inter_frame_delay_.Reset(clock_->TimeInMilliseconds());
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000367 waiting_for_completion_.frame_size = 0;
368 waiting_for_completion_.timestamp = 0;
369 waiting_for_completion_.latest_packet_time = -1;
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000370 first_packet_since_reset_ = true;
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000371 missing_sequence_numbers_.clear();
niklase@google.com470e71d2011-07-07 08:21:25 +0000372}
373
niklase@google.com470e71d2011-07-07 08:21:25 +0000374// Get received key and delta frames
pbos@webrtc.orgce4e9a32014-12-18 13:50:16 +0000375FrameCounts VCMJitterBuffer::FrameStatistics() const {
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000376 CriticalSectionScoped cs(crit_sect_);
sprang@webrtc.org71f055f2013-12-04 15:09:27 +0000377 return receive_statistics_;
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000378}
379
asapersson@webrtc.org96dc6852014-11-03 14:40:38 +0000380int VCMJitterBuffer::num_packets() const {
381 CriticalSectionScoped cs(crit_sect_);
382 return num_packets_;
383}
384
385int VCMJitterBuffer::num_duplicated_packets() const {
386 CriticalSectionScoped cs(crit_sect_);
387 return num_duplicated_packets_;
388}
389
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000390int VCMJitterBuffer::num_discarded_packets() const {
391 CriticalSectionScoped cs(crit_sect_);
392 return num_discarded_packets_;
393}
394
395// Calculate framerate and bitrate.
396void VCMJitterBuffer::IncomingRateStatistics(unsigned int* framerate,
397 unsigned int* bitrate) {
398 assert(framerate);
399 assert(bitrate);
400 CriticalSectionScoped cs(crit_sect_);
stefan@webrtc.orga678a3b2013-01-21 07:42:11 +0000401 const int64_t now = clock_->TimeInMilliseconds();
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000402 int64_t diff = now - time_last_incoming_frame_count_;
403 if (diff < 1000 && incoming_frame_rate_ > 0 && incoming_bit_rate_ > 0) {
404 // Make sure we report something even though less than
405 // 1 second has passed since last update.
406 *framerate = incoming_frame_rate_;
407 *bitrate = incoming_bit_rate_;
408 } else if (incoming_frame_count_ != 0) {
409 // We have received frame(s) since last call to this function
410
411 // Prepare calculations
412 if (diff <= 0) {
413 diff = 1;
niklase@google.com470e71d2011-07-07 08:21:25 +0000414 }
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000415 // we add 0.5f for rounding
416 float rate = 0.5f + ((incoming_frame_count_ * 1000.0f) / diff);
417 if (rate < 1.0f) {
418 rate = 1.0f;
419 }
420
421 // Calculate frame rate
422 // Let r be rate.
423 // r(0) = 1000*framecount/delta_time.
424 // (I.e. frames per second since last calculation.)
425 // frame_rate = r(0)/2 + r(-1)/2
426 // (I.e. fr/s average this and the previous calculation.)
427 *framerate = (incoming_frame_rate_ + static_cast<unsigned int>(rate)) / 2;
428 incoming_frame_rate_ = static_cast<unsigned int>(rate);
429
430 // Calculate bit rate
431 if (incoming_bit_count_ == 0) {
432 *bitrate = 0;
433 } else {
434 *bitrate = 10 * ((100 * incoming_bit_count_) /
435 static_cast<unsigned int>(diff));
436 }
437 incoming_bit_rate_ = *bitrate;
438
439 // Reset count
440 incoming_frame_count_ = 0;
441 incoming_bit_count_ = 0;
442 time_last_incoming_frame_count_ = now;
443
444 } else {
445 // No frames since last call
stefan@webrtc.orga678a3b2013-01-21 07:42:11 +0000446 time_last_incoming_frame_count_ = clock_->TimeInMilliseconds();
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000447 *framerate = 0;
stefan@webrtc.org49806792013-04-30 22:05:07 +0000448 *bitrate = 0;
449 incoming_frame_rate_ = 0;
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000450 incoming_bit_rate_ = 0;
451 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000452}
453
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000454// Answers the question:
455// Will the packet sequence be complete if the next frame is grabbed for
456// decoding right now? That is, have we lost a frame between the last decoded
457// frame and the next, or is the next
458// frame missing one or more packets?
459bool VCMJitterBuffer::CompleteSequenceWithNextFrame() {
460 CriticalSectionScoped cs(crit_sect_);
461 // Finding oldest frame ready for decoder, check sequence number and size
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000462 CleanUpOldOrEmptyFrames();
agalusza@google.comd177c102013-08-08 01:12:33 +0000463 if (!decodable_frames_.empty()) {
464 if (decodable_frames_.Front()->GetState() == kStateComplete) {
465 return true;
466 }
467 } else if (incomplete_frames_.size() <= 1) {
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000468 // Frame not ready to be decoded.
469 return true;
470 }
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000471 return false;
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000472}
473
474// Returns immediately or a |max_wait_time_ms| ms event hang waiting for a
475// complete frame, |max_wait_time_ms| decided by caller.
mikhal@webrtc.org759b0412013-05-07 16:36:00 +0000476bool VCMJitterBuffer::NextCompleteTimestamp(
477 uint32_t max_wait_time_ms, uint32_t* timestamp) {
mikhal@webrtc.orgc1f243f2013-04-22 22:24:38 +0000478 crit_sect_->Enter();
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000479 if (!running_) {
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000480 crit_sect_->Leave();
mikhal@webrtc.org759b0412013-05-07 16:36:00 +0000481 return false;
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000482 }
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000483 CleanUpOldOrEmptyFrames();
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000484
agalusza@google.comd177c102013-08-08 01:12:33 +0000485 if (decodable_frames_.empty() ||
486 decodable_frames_.Front()->GetState() != kStateComplete) {
stefan@webrtc.orga678a3b2013-01-21 07:42:11 +0000487 const int64_t end_wait_time_ms = clock_->TimeInMilliseconds() +
488 max_wait_time_ms;
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000489 int64_t wait_time_ms = max_wait_time_ms;
490 while (wait_time_ms > 0) {
491 crit_sect_->Leave();
492 const EventTypeWrapper ret =
stefan@webrtc.org2baf5f52013-03-13 08:46:25 +0000493 frame_event_->Wait(static_cast<uint32_t>(wait_time_ms));
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000494 crit_sect_->Enter();
495 if (ret == kEventSignaled) {
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000496 // Are we shutting down the jitter buffer?
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000497 if (!running_) {
498 crit_sect_->Leave();
mikhal@webrtc.org759b0412013-05-07 16:36:00 +0000499 return false;
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000500 }
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +0000501 // Finding oldest frame ready for decoder.
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000502 CleanUpOldOrEmptyFrames();
agalusza@google.comd177c102013-08-08 01:12:33 +0000503 if (decodable_frames_.empty() ||
504 decodable_frames_.Front()->GetState() != kStateComplete) {
stefan@webrtc.orga678a3b2013-01-21 07:42:11 +0000505 wait_time_ms = end_wait_time_ms - clock_->TimeInMilliseconds();
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000506 } else {
507 break;
508 }
509 } else {
mikhal@webrtc.org9c7685f2013-05-07 16:07:52 +0000510 break;
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000511 }
512 }
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000513 }
agalusza@google.comd177c102013-08-08 01:12:33 +0000514 if (decodable_frames_.empty() ||
515 decodable_frames_.Front()->GetState() != kStateComplete) {
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000516 crit_sect_->Leave();
517 return false;
mikhal@webrtc.orgdc3cd212013-04-25 20:27:04 +0000518 }
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +0000519 *timestamp = decodable_frames_.Front()->TimeStamp();
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000520 crit_sect_->Leave();
mikhal@webrtc.org759b0412013-05-07 16:36:00 +0000521 return true;
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000522}
523
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000524bool VCMJitterBuffer::NextMaybeIncompleteTimestamp(uint32_t* timestamp) {
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000525 CriticalSectionScoped cs(crit_sect_);
526 if (!running_) {
mikhal@webrtc.org759b0412013-05-07 16:36:00 +0000527 return false;
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000528 }
agalusza@google.coma7e360e2013-08-01 03:15:08 +0000529 if (decode_error_mode_ == kNoErrors) {
mikhal@webrtc.orgdc3cd212013-04-25 20:27:04 +0000530 // No point to continue, as we are not decoding with errors.
mikhal@webrtc.org759b0412013-05-07 16:36:00 +0000531 return false;
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000532 }
533
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000534 CleanUpOldOrEmptyFrames();
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000535
jbauchdb81ffd2015-11-23 03:59:02 -0800536 VCMFrameBuffer* oldest_frame;
agalusza@google.comd177c102013-08-08 01:12:33 +0000537 if (decodable_frames_.empty()) {
jbauchdb81ffd2015-11-23 03:59:02 -0800538 if (nack_mode_ != kNoNack || incomplete_frames_.size() <= 1) {
539 return false;
540 }
541 oldest_frame = incomplete_frames_.Front();
542 // Frame will only be removed from buffer if it is complete (or decodable).
543 if (oldest_frame->GetState() < kStateComplete) {
544 return false;
545 }
546 } else {
547 oldest_frame = decodable_frames_.Front();
548 // If we have exactly one frame in the buffer, release it only if it is
549 // complete. We know decodable_frames_ is not empty due to the previous
550 // check.
551 if (decodable_frames_.size() == 1 && incomplete_frames_.empty()
552 && oldest_frame->GetState() != kStateComplete) {
553 return false;
554 }
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000555 }
mikhal@webrtc.orgd3cd5652013-05-03 17:54:18 +0000556
mikhal@webrtc.org759b0412013-05-07 16:36:00 +0000557 *timestamp = oldest_frame->TimeStamp();
558 return true;
559}
560
561VCMEncodedFrame* VCMJitterBuffer::ExtractAndSetDecode(uint32_t timestamp) {
mikhal@webrtc.org759b0412013-05-07 16:36:00 +0000562 CriticalSectionScoped cs(crit_sect_);
mikhal@webrtc.org759b0412013-05-07 16:36:00 +0000563 if (!running_) {
564 return NULL;
565 }
566 // Extract the frame with the desired timestamp.
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000567 VCMFrameBuffer* frame = decodable_frames_.PopFrame(timestamp);
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +0000568 bool continuous = true;
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000569 if (!frame) {
570 frame = incomplete_frames_.PopFrame(timestamp);
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +0000571 if (frame)
572 continuous = last_decoded_state_.ContinuousFrame(frame);
573 else
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000574 return NULL;
mikhal@webrtc.org759b0412013-05-07 16:36:00 +0000575 }
hclam@chromium.org1a7b9b92013-07-08 21:31:18 +0000576 TRACE_EVENT_ASYNC_STEP0("webrtc", "Video", timestamp, "Extract");
mikhal@webrtc.orgcb20a5b2013-05-15 17:10:44 +0000577 // Frame pulled out from jitter buffer, update the jitter estimate.
mikhal@webrtc.org759b0412013-05-07 16:36:00 +0000578 const bool retransmitted = (frame->GetNackCount() > 0);
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000579 if (retransmitted) {
580 jitter_estimate_.FrameNacked();
mikhal@webrtc.org759b0412013-05-07 16:36:00 +0000581 } else if (frame->Length() > 0) {
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000582 // Ignore retransmitted and empty frames.
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000583 if (waiting_for_completion_.latest_packet_time >= 0) {
584 UpdateJitterEstimate(waiting_for_completion_, true);
585 }
mikhal@webrtc.orgcb20a5b2013-05-15 17:10:44 +0000586 if (frame->GetState() == kStateComplete) {
587 UpdateJitterEstimate(*frame, false);
588 } else {
589 // Wait for this one to get complete.
590 waiting_for_completion_.frame_size = frame->Length();
591 waiting_for_completion_.latest_packet_time =
592 frame->LatestPacketTimeMs();
593 waiting_for_completion_.timestamp = frame->TimeStamp();
594 }
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000595 }
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000596
597 // The state must be changed to decoding before cleaning up zero sized
598 // frames to avoid empty frames being cleaned up and then given to the
mikhal@webrtc.orgcb20a5b2013-05-15 17:10:44 +0000599 // decoder. Propagates the missing_frame bit.
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +0000600 frame->PrepareForDecode(continuous);
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000601
mikhal@webrtc.orgcb20a5b2013-05-15 17:10:44 +0000602 // We have a frame - update the last decoded state and nack list.
mikhal@webrtc.org759b0412013-05-07 16:36:00 +0000603 last_decoded_state_.SetState(frame);
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000604 DropPacketsFromNackList(last_decoded_state_.sequence_num());
agalusza@google.comd818dcb2013-07-29 21:48:11 +0000605
606 if ((*frame).IsSessionComplete())
607 UpdateAveragePacketsPerFrame(frame->NumPackets());
608
mikhal@webrtc.org759b0412013-05-07 16:36:00 +0000609 return frame;
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000610}
611
612// Release frame when done with decoding. Should never be used to release
613// frames from within the jitter buffer.
614void VCMJitterBuffer::ReleaseFrame(VCMEncodedFrame* frame) {
615 CriticalSectionScoped cs(crit_sect_);
616 VCMFrameBuffer* frame_buffer = static_cast<VCMFrameBuffer*>(frame);
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +0000617 if (frame_buffer) {
618 free_frames_.push_back(frame_buffer);
619 }
stefan@webrtc.org791eec72011-10-11 07:53:43 +0000620}
621
niklase@google.com470e71d2011-07-07 08:21:25 +0000622// Gets frame to use for this timestamp. If no match, get empty frame.
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000623VCMFrameBufferEnum VCMJitterBuffer::GetFrame(const VCMPacket& packet,
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000624 VCMFrameBuffer** frame,
625 FrameList** frame_list) {
626 *frame = incomplete_frames_.PopFrame(packet.timestamp);
627 if (*frame != NULL) {
628 *frame_list = &incomplete_frames_;
629 return kNoError;
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000630 }
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000631 *frame = decodable_frames_.PopFrame(packet.timestamp);
632 if (*frame != NULL) {
633 *frame_list = &decodable_frames_;
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000634 return kNoError;
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000635 }
mikhal@webrtc.org3c5a9242013-09-03 20:45:36 +0000636
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000637 *frame_list = NULL;
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000638 // No match, return empty frame.
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000639 *frame = GetEmptyFrame();
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000640 if (*frame == NULL) {
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +0000641 // No free frame! Try to reclaim some...
stefan@webrtc.org34c5da62014-04-11 14:08:35 +0000642 LOG(LS_WARNING) << "Unable to get empty frame; Recycling.";
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +0000643 bool found_key_frame = RecycleFramesUntilKeyFrame();
644 *frame = GetEmptyFrame();
stefan@webrtc.org34c5da62014-04-11 14:08:35 +0000645 assert(*frame);
646 if (!found_key_frame) {
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000647 free_frames_.push_back(*frame);
648 return kFlushIndicator;
stefan@webrtc.org34c5da62014-04-11 14:08:35 +0000649 }
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000650 }
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +0000651 (*frame)->Reset();
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000652 return kNoError;
niklase@google.com470e71d2011-07-07 08:21:25 +0000653}
654
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000655int64_t VCMJitterBuffer::LastPacketTime(const VCMEncodedFrame* frame,
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000656 bool* retransmitted) const {
657 assert(retransmitted);
658 CriticalSectionScoped cs(crit_sect_);
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000659 const VCMFrameBuffer* frame_buffer =
660 static_cast<const VCMFrameBuffer*>(frame);
661 *retransmitted = (frame_buffer->GetNackCount() > 0);
662 return frame_buffer->LatestPacketTimeMs();
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000663}
664
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000665VCMFrameBufferEnum VCMJitterBuffer::InsertPacket(const VCMPacket& packet,
666 bool* retransmitted) {
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000667 CriticalSectionScoped cs(crit_sect_);
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +0000668
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000669 ++num_packets_;
asapersson@webrtc.org83b52002014-11-28 10:17:13 +0000670 if (num_packets_ == 1) {
671 time_first_packet_ms_ = clock_->TimeInMilliseconds();
672 }
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000673 // Does this packet belong to an old frame?
674 if (last_decoded_state_.IsOldPacket(&packet)) {
675 // Account only for media packets.
676 if (packet.sizeBytes > 0) {
677 num_discarded_packets_++;
678 num_consecutive_old_packets_++;
pbos@webrtc.org55707692014-12-19 15:45:03 +0000679 if (stats_callback_ != NULL)
680 stats_callback_->OnDiscardedPacketsUpdated(num_discarded_packets_);
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000681 }
682 // Update last decoded sequence number if the packet arrived late and
683 // belongs to a frame with a timestamp equal to the last decoded
684 // timestamp.
685 last_decoded_state_.UpdateOldPacket(&packet);
686 DropPacketsFromNackList(last_decoded_state_.sequence_num());
687
Noah Richardse4cb4e92015-05-22 14:03:00 -0700688 // Also see if this old packet made more incomplete frames continuous.
689 FindAndInsertContinuousFramesWithState(last_decoded_state_);
690
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000691 if (num_consecutive_old_packets_ > kMaxConsecutiveOldPackets) {
692 LOG(LS_WARNING)
693 << num_consecutive_old_packets_
694 << " consecutive old packets received. Flushing the jitter buffer.";
695 Flush();
696 return kFlushIndicator;
697 }
698 return kOldPacket;
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000699 }
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000700
asapersson9a4cd872015-10-23 00:27:14 -0700701 num_consecutive_old_packets_ = 0;
702
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000703 VCMFrameBuffer* frame;
704 FrameList* frame_list;
705 const VCMFrameBufferEnum error = GetFrame(packet, &frame, &frame_list);
706 if (error != kNoError)
707 return error;
708
mikhal@webrtc.org3c5a9242013-09-03 20:45:36 +0000709 int64_t now_ms = clock_->TimeInMilliseconds();
710 // We are keeping track of the first and latest seq numbers, and
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000711 // the number of wraps to be able to calculate how many packets we expect.
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000712 if (first_packet_since_reset_) {
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000713 // Now it's time to start estimating jitter
714 // reset the delay estimate.
mikhal@webrtc.org3c5a9242013-09-03 20:45:36 +0000715 inter_frame_delay_.Reset(now_ms);
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000716 }
717
718 // Empty packets may bias the jitter estimate (lacking size component),
719 // therefore don't let empty packet trigger the following updates:
pbos22993e12015-10-19 02:39:06 -0700720 if (packet.frameType != kEmptyFrame) {
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000721 if (waiting_for_completion_.timestamp == packet.timestamp) {
722 // This can get bad if we have a lot of duplicate packets,
723 // we will then count some packet multiple times.
724 waiting_for_completion_.frame_size += packet.sizeBytes;
725 waiting_for_completion_.latest_packet_time = now_ms;
726 } else if (waiting_for_completion_.latest_packet_time >= 0 &&
727 waiting_for_completion_.latest_packet_time + 2000 <= now_ms) {
728 // A packet should never be more than two seconds late
729 UpdateJitterEstimate(waiting_for_completion_, true);
730 waiting_for_completion_.latest_packet_time = -1;
731 waiting_for_completion_.frame_size = 0;
732 waiting_for_completion_.timestamp = 0;
733 }
734 }
735
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000736 VCMFrameBufferStateEnum previous_state = frame->GetState();
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000737 // Insert packet.
agalusza@google.comd818dcb2013-07-29 21:48:11 +0000738 FrameData frame_data;
739 frame_data.rtt_ms = rtt_ms_;
740 frame_data.rolling_average_packets_per_frame = average_packets_per_frame_;
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000741 VCMFrameBufferEnum buffer_state =
742 frame->InsertPacket(packet, now_ms, decode_error_mode_, frame_data);
743
744 if (previous_state != kStateComplete) {
hclam@chromium.org1a7b9b92013-07-08 21:31:18 +0000745 TRACE_EVENT_ASYNC_BEGIN1("webrtc", "Video", frame->TimeStamp(),
746 "timestamp", frame->TimeStamp());
747 }
mikhal@webrtc.org3c5a9242013-09-03 20:45:36 +0000748
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000749 if (buffer_state > 0) {
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000750 incoming_bit_count_ += packet.sizeBytes << 3;
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000751 if (first_packet_since_reset_) {
752 latest_received_sequence_number_ = packet.seqNum;
753 first_packet_since_reset_ = false;
754 } else {
755 if (IsPacketRetransmitted(packet)) {
756 frame->IncrementNackCount();
757 }
pbos@webrtc.orgebb467f2014-05-19 15:28:02 +0000758 if (!UpdateNackList(packet.seqNum) &&
759 packet.frameType != kVideoFrameKey) {
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000760 buffer_state = kFlushIndicator;
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000761 }
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000762
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000763 latest_received_sequence_number_ = LatestSequenceNumber(
764 latest_received_sequence_number_, packet.seqNum);
765 }
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000766 }
mikhal@webrtc.org3c5a9242013-09-03 20:45:36 +0000767
768 // Is the frame already in the decodable list?
mikhal@webrtc.org3c5a9242013-09-03 20:45:36 +0000769 bool continuous = IsContinuous(*frame);
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000770 switch (buffer_state) {
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000771 case kGeneralError:
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000772 case kTimeStampError:
773 case kSizeError: {
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000774 free_frames_.push_back(frame);
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000775 break;
776 }
777 case kCompleteSession: {
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000778 if (previous_state != kStateDecodable &&
779 previous_state != kStateComplete) {
mikhal@webrtc.org3c5a9242013-09-03 20:45:36 +0000780 CountFrame(*frame);
mikhal@webrtc.org3c5a9242013-09-03 20:45:36 +0000781 if (continuous) {
782 // Signal that we have a complete session.
783 frame_event_->Set();
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000784 }
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000785 }
kjellander@webrtc.org7d2b6a92015-01-28 18:37:58 +0000786 FALLTHROUGH();
mikhal@webrtc.org3c5a9242013-09-03 20:45:36 +0000787 }
788 // Note: There is no break here - continuing to kDecodableSession.
789 case kDecodableSession: {
790 *retransmitted = (frame->GetNackCount() > 0);
mikhal@webrtc.org3c5a9242013-09-03 20:45:36 +0000791 if (continuous) {
mikhal@webrtc.org3c5a9242013-09-03 20:45:36 +0000792 decodable_frames_.InsertFrame(frame);
793 FindAndInsertContinuousFrames(*frame);
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000794 } else {
mikhal@webrtc.org3c5a9242013-09-03 20:45:36 +0000795 incomplete_frames_.InsertFrame(frame);
jbauchdb81ffd2015-11-23 03:59:02 -0800796 // If NACKs are enabled, keyframes are triggered by |GetNackList|.
797 if (nack_mode_ == kNoNack && NonContinuousOrIncompleteDuration() >
798 90 * kMaxDiscontinuousFramesTime) {
799 return kFlushIndicator;
800 }
mikhal@webrtc.org3c5a9242013-09-03 20:45:36 +0000801 }
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000802 break;
803 }
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000804 case kIncomplete: {
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000805 if (frame->GetState() == kStateEmpty &&
806 last_decoded_state_.UpdateEmptyFrame(frame)) {
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +0000807 free_frames_.push_back(frame);
mikhal@webrtc.org3c5a9242013-09-03 20:45:36 +0000808 return kNoError;
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000809 } else {
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000810 incomplete_frames_.InsertFrame(frame);
jbauchdb81ffd2015-11-23 03:59:02 -0800811 // If NACKs are enabled, keyframes are triggered by |GetNackList|.
812 if (nack_mode_ == kNoNack && NonContinuousOrIncompleteDuration() >
813 90 * kMaxDiscontinuousFramesTime) {
814 return kFlushIndicator;
815 }
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000816 }
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000817 break;
818 }
819 case kNoError:
mikhal@webrtc.orgf31a47a2013-08-26 17:10:11 +0000820 case kOutOfBoundsPacket:
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000821 case kDuplicatePacket: {
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000822 // Put back the frame where it came from.
823 if (frame_list != NULL) {
824 frame_list->InsertFrame(frame);
825 } else {
826 free_frames_.push_back(frame);
827 }
asapersson@webrtc.org96dc6852014-11-03 14:40:38 +0000828 ++num_duplicated_packets_;
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000829 break;
830 }
stefan@webrtc.org3417eb42013-05-21 15:25:53 +0000831 case kFlushIndicator:
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000832 free_frames_.push_back(frame);
mikhal@webrtc.org3c5a9242013-09-03 20:45:36 +0000833 return kFlushIndicator;
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000834 default: assert(false);
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000835 }
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000836 return buffer_state;
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000837}
838
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000839bool VCMJitterBuffer::IsContinuousInState(const VCMFrameBuffer& frame,
840 const VCMDecodingState& decoding_state) const {
agalusza@google.coma7e360e2013-08-01 03:15:08 +0000841 // Is this frame (complete or decodable) and continuous?
842 // kStateDecodable will never be set when decode_error_mode_ is false
843 // as SessionInfo determines this state based on the error mode (and frame
844 // completeness).
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000845 return (frame.GetState() == kStateComplete ||
846 frame.GetState() == kStateDecodable) &&
847 decoding_state.ContinuousFrame(&frame);
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000848}
849
850bool VCMJitterBuffer::IsContinuous(const VCMFrameBuffer& frame) const {
851 if (IsContinuousInState(frame, last_decoded_state_)) {
852 return true;
853 }
854 VCMDecodingState decoding_state;
855 decoding_state.CopyFrom(last_decoded_state_);
856 for (FrameList::const_iterator it = decodable_frames_.begin();
857 it != decodable_frames_.end(); ++it) {
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +0000858 VCMFrameBuffer* decodable_frame = it->second;
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000859 if (IsNewerTimestamp(decodable_frame->TimeStamp(), frame.TimeStamp())) {
860 break;
861 }
862 decoding_state.SetState(decodable_frame);
863 if (IsContinuousInState(frame, decoding_state)) {
864 return true;
865 }
866 }
867 return false;
868}
869
870void VCMJitterBuffer::FindAndInsertContinuousFrames(
871 const VCMFrameBuffer& new_frame) {
872 VCMDecodingState decoding_state;
873 decoding_state.CopyFrom(last_decoded_state_);
874 decoding_state.SetState(&new_frame);
Noah Richardse4cb4e92015-05-22 14:03:00 -0700875 FindAndInsertContinuousFramesWithState(decoding_state);
876}
877
878void VCMJitterBuffer::FindAndInsertContinuousFramesWithState(
879 const VCMDecodingState& original_decoded_state) {
880 // Copy original_decoded_state so we can move the state forward with each
881 // decodable frame we find.
882 VCMDecodingState decoding_state;
883 decoding_state.CopyFrom(original_decoded_state);
884
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000885 // When temporal layers are available, we search for a complete or decodable
886 // frame until we hit one of the following:
887 // 1. Continuous base or sync layer.
888 // 2. The end of the list was reached.
889 for (FrameList::iterator it = incomplete_frames_.begin();
890 it != incomplete_frames_.end();) {
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +0000891 VCMFrameBuffer* frame = it->second;
Noah Richardse4cb4e92015-05-22 14:03:00 -0700892 if (IsNewerTimestamp(original_decoded_state.time_stamp(),
893 frame->TimeStamp())) {
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000894 ++it;
895 continue;
896 }
897 if (IsContinuousInState(*frame, decoding_state)) {
898 decodable_frames_.InsertFrame(frame);
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +0000899 incomplete_frames_.erase(it++);
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000900 decoding_state.SetState(frame);
901 } else if (frame->TemporalId() <= 0) {
902 break;
903 } else {
904 ++it;
905 }
906 }
907}
908
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000909uint32_t VCMJitterBuffer::EstimatedJitterMs() {
910 CriticalSectionScoped cs(crit_sect_);
mikhal@webrtc.org119c67d2013-01-31 17:18:02 +0000911 // Compute RTT multiplier for estimation.
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000912 // low_rtt_nackThresholdMs_ == -1 means no FEC.
913 double rtt_mult = 1.0f;
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000914 if (low_rtt_nack_threshold_ms_ >= 0 &&
pkasting@chromium.org16825b12015-01-12 21:51:21 +0000915 rtt_ms_ >= low_rtt_nack_threshold_ms_) {
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000916 // For RTTs above low_rtt_nack_threshold_ms_ we don't apply extra delay
917 // when waiting for retransmissions.
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000918 rtt_mult = 0.0f;
919 }
mikhal@webrtc.org119c67d2013-01-31 17:18:02 +0000920 return jitter_estimate_.GetJitterEstimate(rtt_mult);
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000921}
922
pkasting@chromium.org16825b12015-01-12 21:51:21 +0000923void VCMJitterBuffer::UpdateRtt(int64_t rtt_ms) {
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000924 CriticalSectionScoped cs(crit_sect_);
925 rtt_ms_ = rtt_ms;
926 jitter_estimate_.UpdateRtt(rtt_ms);
927}
928
929void VCMJitterBuffer::SetNackMode(VCMNackMode mode,
pkasting@chromium.org16825b12015-01-12 21:51:21 +0000930 int64_t low_rtt_nack_threshold_ms,
931 int64_t high_rtt_nack_threshold_ms) {
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000932 CriticalSectionScoped cs(crit_sect_);
933 nack_mode_ = mode;
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000934 if (mode == kNoNack) {
935 missing_sequence_numbers_.clear();
936 }
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000937 assert(low_rtt_nack_threshold_ms >= -1 && high_rtt_nack_threshold_ms >= -1);
938 assert(high_rtt_nack_threshold_ms == -1 ||
939 low_rtt_nack_threshold_ms <= high_rtt_nack_threshold_ms);
940 assert(low_rtt_nack_threshold_ms > -1 || high_rtt_nack_threshold_ms == -1);
941 low_rtt_nack_threshold_ms_ = low_rtt_nack_threshold_ms;
942 high_rtt_nack_threshold_ms_ = high_rtt_nack_threshold_ms;
mflodman@webrtc.org7c894b72012-11-26 12:40:15 +0000943 // Don't set a high start rtt if high_rtt_nack_threshold_ms_ is used, to not
Wan-Teh Changf2912872015-06-05 13:16:45 -0700944 // disable NACK in |kNack| mode.
mflodman@webrtc.org7c894b72012-11-26 12:40:15 +0000945 if (rtt_ms_ == kDefaultRtt && high_rtt_nack_threshold_ms_ != -1) {
946 rtt_ms_ = 0;
947 }
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000948 if (!WaitForRetransmissions()) {
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000949 jitter_estimate_.ResetNackCount();
950 }
951}
952
stefan@webrtc.orgbecf9c82013-02-01 15:09:57 +0000953void VCMJitterBuffer::SetNackSettings(size_t max_nack_list_size,
stefan@webrtc.orgef144882013-05-07 19:16:33 +0000954 int max_packet_age_to_nack,
955 int max_incomplete_time_ms) {
stefan@webrtc.orgbecf9c82013-02-01 15:09:57 +0000956 CriticalSectionScoped cs(crit_sect_);
957 assert(max_packet_age_to_nack >= 0);
stefan@webrtc.orgef144882013-05-07 19:16:33 +0000958 assert(max_incomplete_time_ms_ >= 0);
stefan@webrtc.orgbecf9c82013-02-01 15:09:57 +0000959 max_nack_list_size_ = max_nack_list_size;
960 max_packet_age_to_nack_ = max_packet_age_to_nack;
stefan@webrtc.orgef144882013-05-07 19:16:33 +0000961 max_incomplete_time_ms_ = max_incomplete_time_ms;
stefan@webrtc.orgbecf9c82013-02-01 15:09:57 +0000962}
963
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000964VCMNackMode VCMJitterBuffer::nack_mode() const {
965 CriticalSectionScoped cs(crit_sect_);
966 return nack_mode_;
967}
968
stefan@webrtc.orgef144882013-05-07 19:16:33 +0000969int VCMJitterBuffer::NonContinuousOrIncompleteDuration() {
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000970 if (incomplete_frames_.empty()) {
stefan@webrtc.orgef144882013-05-07 19:16:33 +0000971 return 0;
972 }
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +0000973 uint32_t start_timestamp = incomplete_frames_.Front()->TimeStamp();
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000974 if (!decodable_frames_.empty()) {
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +0000975 start_timestamp = decodable_frames_.Back()->TimeStamp();
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +0000976 }
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +0000977 return incomplete_frames_.Back()->TimeStamp() - start_timestamp;
stefan@webrtc.orgef144882013-05-07 19:16:33 +0000978}
979
980uint16_t VCMJitterBuffer::EstimatedLowSequenceNumber(
981 const VCMFrameBuffer& frame) const {
982 assert(frame.GetLowSeqNum() >= 0);
983 if (frame.HaveFirstPacket())
984 return frame.GetLowSeqNum();
hclam@chromium.orgfe307e12013-05-16 21:19:59 +0000985
986 // This estimate is not accurate if more than one packet with lower sequence
987 // number is lost.
stefan@webrtc.orgef144882013-05-07 19:16:33 +0000988 return frame.GetLowSeqNum() - 1;
989}
990
Wan-Teh Changb1825a42015-06-03 15:03:35 -0700991std::vector<uint16_t> VCMJitterBuffer::GetNackList(bool* request_key_frame) {
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000992 CriticalSectionScoped cs(crit_sect_);
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000993 *request_key_frame = false;
994 if (nack_mode_ == kNoNack) {
Wan-Teh Changb1825a42015-06-03 15:03:35 -0700995 return std::vector<uint16_t>();
stefan@webrtc.org912981f2012-10-12 07:04:52 +0000996 }
stefan@webrtc.orga64300a2013-03-04 15:24:40 +0000997 if (last_decoded_state_.in_initial_state()) {
pbos@webrtc.org4f16c872014-11-24 09:06:48 +0000998 VCMFrameBuffer* next_frame = NextFrame();
agalusza@google.comd177c102013-08-08 01:12:33 +0000999 const bool first_frame_is_key = next_frame &&
1000 next_frame->FrameType() == kVideoFrameKey &&
1001 next_frame->HaveFirstPacket();
stefan@webrtc.org885cd132013-04-16 09:38:26 +00001002 if (!first_frame_is_key) {
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +00001003 bool have_non_empty_frame = decodable_frames_.end() != find_if(
1004 decodable_frames_.begin(), decodable_frames_.end(),
1005 HasNonEmptyState);
1006 if (!have_non_empty_frame) {
1007 have_non_empty_frame = incomplete_frames_.end() != find_if(
1008 incomplete_frames_.begin(), incomplete_frames_.end(),
1009 HasNonEmptyState);
1010 }
stefan@webrtc.org885cd132013-04-16 09:38:26 +00001011 bool found_key_frame = RecycleFramesUntilKeyFrame();
1012 if (!found_key_frame) {
1013 *request_key_frame = have_non_empty_frame;
Wan-Teh Changb1825a42015-06-03 15:03:35 -07001014 return std::vector<uint16_t>();
stefan@webrtc.org885cd132013-04-16 09:38:26 +00001015 }
1016 }
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001017 }
stefan@webrtc.orga64300a2013-03-04 15:24:40 +00001018 if (TooLargeNackList()) {
1019 *request_key_frame = !HandleTooLargeNackList();
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001020 }
stefan@webrtc.orgef144882013-05-07 19:16:33 +00001021 if (max_incomplete_time_ms_ > 0) {
1022 int non_continuous_incomplete_duration =
1023 NonContinuousOrIncompleteDuration();
1024 if (non_continuous_incomplete_duration > 90 * max_incomplete_time_ms_) {
stefan@webrtc.org34c5da62014-04-11 14:08:35 +00001025 LOG_F(LS_WARNING) << "Too long non-decodable duration: "
1026 << non_continuous_incomplete_duration << " > "
1027 << 90 * max_incomplete_time_ms_;
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +00001028 FrameList::reverse_iterator rit = find_if(incomplete_frames_.rbegin(),
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +00001029 incomplete_frames_.rend(), IsKeyFrame);
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +00001030 if (rit == incomplete_frames_.rend()) {
stefan@webrtc.orgef144882013-05-07 19:16:33 +00001031 // Request a key frame if we don't have one already.
1032 *request_key_frame = true;
Wan-Teh Changb1825a42015-06-03 15:03:35 -07001033 return std::vector<uint16_t>();
stefan@webrtc.orgef144882013-05-07 19:16:33 +00001034 } else {
1035 // Skip to the last key frame. If it's incomplete we will start
1036 // NACKing it.
hclam@chromium.orgfe307e12013-05-16 21:19:59 +00001037 // Note that the estimated low sequence number is correct for VP8
1038 // streams because only the first packet of a key frame is marked.
stefan@webrtc.orgef144882013-05-07 19:16:33 +00001039 last_decoded_state_.Reset();
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +00001040 DropPacketsFromNackList(EstimatedLowSequenceNumber(*rit->second));
stefan@webrtc.orgef144882013-05-07 19:16:33 +00001041 }
1042 }
1043 }
Wan-Teh Changb1825a42015-06-03 15:03:35 -07001044 std::vector<uint16_t> nack_list(missing_sequence_numbers_.begin(),
1045 missing_sequence_numbers_.end());
1046 return nack_list;
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001047}
1048
mikhal@webrtc.orgdbf6a812013-08-21 20:40:47 +00001049void VCMJitterBuffer::SetDecodeErrorMode(VCMDecodeErrorMode error_mode) {
1050 CriticalSectionScoped cs(crit_sect_);
mikhal@webrtc.org3c5a9242013-09-03 20:45:36 +00001051 decode_error_mode_ = error_mode;
agalusza@google.comd177c102013-08-08 01:12:33 +00001052}
1053
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +00001054VCMFrameBuffer* VCMJitterBuffer::NextFrame() const {
1055 if (!decodable_frames_.empty())
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +00001056 return decodable_frames_.Front();
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +00001057 if (!incomplete_frames_.empty())
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +00001058 return incomplete_frames_.Front();
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +00001059 return NULL;
1060}
1061
stefan@webrtc.orga64300a2013-03-04 15:24:40 +00001062bool VCMJitterBuffer::UpdateNackList(uint16_t sequence_number) {
1063 if (nack_mode_ == kNoNack) {
1064 return true;
1065 }
stefan@webrtc.org4d2f5de2013-04-09 18:24:41 +00001066 // Make sure we don't add packets which are already too old to be decoded.
stefan@webrtc.orga64300a2013-03-04 15:24:40 +00001067 if (!last_decoded_state_.in_initial_state()) {
stefan@webrtc.orga64300a2013-03-04 15:24:40 +00001068 latest_received_sequence_number_ = LatestSequenceNumber(
1069 latest_received_sequence_number_,
stefan@webrtc.org7bc465b2013-04-11 17:48:02 +00001070 last_decoded_state_.sequence_num());
stefan@webrtc.org4d2f5de2013-04-09 18:24:41 +00001071 }
stefan@webrtc.org7bc465b2013-04-11 17:48:02 +00001072 if (IsNewerSequenceNumber(sequence_number,
1073 latest_received_sequence_number_)) {
stefan@webrtc.org4d2f5de2013-04-09 18:24:41 +00001074 // Push any missing sequence numbers to the NACK list.
1075 for (uint16_t i = latest_received_sequence_number_ + 1;
stefan@webrtc.orga5dee332013-05-07 11:11:17 +00001076 IsNewerSequenceNumber(sequence_number, i); ++i) {
stefan@webrtc.org4d2f5de2013-04-09 18:24:41 +00001077 missing_sequence_numbers_.insert(missing_sequence_numbers_.end(), i);
sprang@webrtc.org0200f702015-02-16 12:06:00 +00001078 TRACE_EVENT_INSTANT1(TRACE_DISABLED_BY_DEFAULT("webrtc_rtp"), "AddNack",
1079 "seqnum", i);
stefan@webrtc.orga64300a2013-03-04 15:24:40 +00001080 }
stefan@webrtc.org4d2f5de2013-04-09 18:24:41 +00001081 if (TooLargeNackList() && !HandleTooLargeNackList()) {
stefan@webrtc.org34c5da62014-04-11 14:08:35 +00001082 LOG(LS_WARNING) << "Requesting key frame due to too large NACK list.";
stefan@webrtc.org4d2f5de2013-04-09 18:24:41 +00001083 return false;
1084 }
1085 if (MissingTooOldPacket(sequence_number) &&
1086 !HandleTooOldPackets(sequence_number)) {
stefan@webrtc.org34c5da62014-04-11 14:08:35 +00001087 LOG(LS_WARNING) << "Requesting key frame due to missing too old packets";
stefan@webrtc.org4d2f5de2013-04-09 18:24:41 +00001088 return false;
1089 }
1090 } else {
1091 missing_sequence_numbers_.erase(sequence_number);
sprang@webrtc.org0200f702015-02-16 12:06:00 +00001092 TRACE_EVENT_INSTANT1(TRACE_DISABLED_BY_DEFAULT("webrtc_rtp"), "RemoveNack",
1093 "seqnum", sequence_number);
stefan@webrtc.orga64300a2013-03-04 15:24:40 +00001094 }
1095 return true;
1096}
1097
1098bool VCMJitterBuffer::TooLargeNackList() const {
1099 return missing_sequence_numbers_.size() > max_nack_list_size_;
1100}
1101
1102bool VCMJitterBuffer::HandleTooLargeNackList() {
1103 // Recycle frames until the NACK list is small enough. It is likely cheaper to
1104 // request a key frame than to retransmit this many missing packets.
stefan@webrtc.org34c5da62014-04-11 14:08:35 +00001105 LOG_F(LS_WARNING) << "NACK list has grown too large: "
1106 << missing_sequence_numbers_.size() << " > "
1107 << max_nack_list_size_;
stefan@webrtc.orga64300a2013-03-04 15:24:40 +00001108 bool key_frame_found = false;
stefan@webrtc.org4d2f5de2013-04-09 18:24:41 +00001109 while (TooLargeNackList()) {
stefan@webrtc.orga64300a2013-03-04 15:24:40 +00001110 key_frame_found = RecycleFramesUntilKeyFrame();
1111 }
stefan@webrtc.orga64300a2013-03-04 15:24:40 +00001112 return key_frame_found;
1113}
1114
1115bool VCMJitterBuffer::MissingTooOldPacket(
1116 uint16_t latest_sequence_number) const {
1117 if (missing_sequence_numbers_.empty()) {
1118 return false;
1119 }
1120 const uint16_t age_of_oldest_missing_packet = latest_sequence_number -
1121 *missing_sequence_numbers_.begin();
1122 // Recycle frames if the NACK list contains too old sequence numbers as
1123 // the packets may have already been dropped by the sender.
1124 return age_of_oldest_missing_packet > max_packet_age_to_nack_;
1125}
1126
1127bool VCMJitterBuffer::HandleTooOldPackets(uint16_t latest_sequence_number) {
1128 bool key_frame_found = false;
1129 const uint16_t age_of_oldest_missing_packet = latest_sequence_number -
1130 *missing_sequence_numbers_.begin();
stefan@webrtc.org34c5da62014-04-11 14:08:35 +00001131 LOG_F(LS_WARNING) << "NACK list contains too old sequence numbers: "
1132 << age_of_oldest_missing_packet << " > "
1133 << max_packet_age_to_nack_;
stefan@webrtc.orga64300a2013-03-04 15:24:40 +00001134 while (MissingTooOldPacket(latest_sequence_number)) {
1135 key_frame_found = RecycleFramesUntilKeyFrame();
1136 }
stefan@webrtc.orga64300a2013-03-04 15:24:40 +00001137 return key_frame_found;
1138}
1139
1140void VCMJitterBuffer::DropPacketsFromNackList(
1141 uint16_t last_decoded_sequence_number) {
1142 // Erase all sequence numbers from the NACK list which we won't need any
1143 // longer.
1144 missing_sequence_numbers_.erase(missing_sequence_numbers_.begin(),
1145 missing_sequence_numbers_.upper_bound(
1146 last_decoded_sequence_number));
1147}
1148
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001149int64_t VCMJitterBuffer::LastDecodedTimestamp() const {
1150 CriticalSectionScoped cs(crit_sect_);
1151 return last_decoded_state_.time_stamp();
1152}
1153
stefan@webrtc.orgef144882013-05-07 19:16:33 +00001154void VCMJitterBuffer::RenderBufferSize(uint32_t* timestamp_start,
1155 uint32_t* timestamp_end) {
1156 CriticalSectionScoped cs(crit_sect_);
1157 CleanUpOldOrEmptyFrames();
1158 *timestamp_start = 0;
1159 *timestamp_end = 0;
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +00001160 if (decodable_frames_.empty()) {
stefan@webrtc.orgef144882013-05-07 19:16:33 +00001161 return;
1162 }
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +00001163 *timestamp_start = decodable_frames_.Front()->TimeStamp();
1164 *timestamp_end = decodable_frames_.Back()->TimeStamp();
mikhal@webrtc.org381da4b2013-04-25 21:45:29 +00001165}
1166
pbos@webrtc.org55707692014-12-19 15:45:03 +00001167void VCMJitterBuffer::RegisterStatsCallback(
1168 VCMReceiveStatisticsCallback* callback) {
pbos@webrtc.orgce4e9a32014-12-18 13:50:16 +00001169 CriticalSectionScoped cs(crit_sect_);
pbos@webrtc.org55707692014-12-19 15:45:03 +00001170 stats_callback_ = callback;
pbos@webrtc.orgce4e9a32014-12-18 13:50:16 +00001171}
1172
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001173VCMFrameBuffer* VCMJitterBuffer::GetEmptyFrame() {
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +00001174 if (free_frames_.empty()) {
1175 if (!TryToIncreaseJitterBufferSize()) {
1176 return NULL;
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001177 }
1178 }
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +00001179 VCMFrameBuffer* frame = free_frames_.front();
1180 free_frames_.pop_front();
1181 return frame;
1182}
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001183
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +00001184bool VCMJitterBuffer::TryToIncreaseJitterBufferSize() {
1185 if (max_number_of_frames_ >= kMaxNumberOfFrames)
1186 return false;
pbos@webrtc.org4f16c872014-11-24 09:06:48 +00001187 free_frames_.push_back(new VCMFrameBuffer());
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +00001188 ++max_number_of_frames_;
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +00001189 TRACE_COUNTER1("webrtc", "JBMaxFrames", max_number_of_frames_);
1190 return true;
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001191}
1192
1193// Recycle oldest frames up to a key frame, used if jitter buffer is completely
1194// full.
1195bool VCMJitterBuffer::RecycleFramesUntilKeyFrame() {
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +00001196 // First release incomplete frames, and only release decodable frames if there
1197 // are no incomplete ones.
1198 FrameList::iterator key_frame_it;
1199 bool key_frame_found = false;
1200 int dropped_frames = 0;
1201 dropped_frames += incomplete_frames_.RecycleFramesUntilKeyFrame(
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +00001202 &key_frame_it, &free_frames_);
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +00001203 key_frame_found = key_frame_it != incomplete_frames_.end();
1204 if (dropped_frames == 0) {
1205 dropped_frames += decodable_frames_.RecycleFramesUntilKeyFrame(
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +00001206 &key_frame_it, &free_frames_);
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +00001207 key_frame_found = key_frame_it != decodable_frames_.end();
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001208 }
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +00001209 TRACE_EVENT_INSTANT0("webrtc", "JB::RecycleFramesUntilKeyFrame");
1210 if (key_frame_found) {
stefan@webrtc.org34c5da62014-04-11 14:08:35 +00001211 LOG(LS_INFO) << "Found key frame while dropping frames.";
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +00001212 // Reset last decoded state to make sure the next frame decoded is a key
1213 // frame, and start NACKing from here.
1214 last_decoded_state_.Reset();
stefan@webrtc.org50fb4af2013-06-17 07:33:58 +00001215 DropPacketsFromNackList(EstimatedLowSequenceNumber(*key_frame_it->second));
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +00001216 } else if (decodable_frames_.empty()) {
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +00001217 // All frames dropped. Reset the decoding state and clear missing sequence
1218 // numbers as we're starting fresh.
1219 last_decoded_state_.Reset();
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +00001220 missing_sequence_numbers_.clear();
edjee@google.com79b02892013-04-04 19:43:34 +00001221 }
stefan@webrtc.orgc8b29a22013-06-17 07:13:16 +00001222 return key_frame_found;
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001223}
1224
1225// Must be called under the critical section |crit_sect_|.
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +00001226void VCMJitterBuffer::CountFrame(const VCMFrameBuffer& frame) {
pbos@webrtc.org4f16c872014-11-24 09:06:48 +00001227 incoming_frame_count_++;
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001228
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +00001229 if (frame.FrameType() == kVideoFrameKey) {
hclam@chromium.org1a7b9b92013-07-08 21:31:18 +00001230 TRACE_EVENT_ASYNC_STEP0("webrtc", "Video",
1231 frame.TimeStamp(), "KeyComplete");
hclam@chromium.org806dc3b2013-04-09 19:54:10 +00001232 } else {
hclam@chromium.org1a7b9b92013-07-08 21:31:18 +00001233 TRACE_EVENT_ASYNC_STEP0("webrtc", "Video",
1234 frame.TimeStamp(), "DeltaComplete");
hclam@chromium.org806dc3b2013-04-09 19:54:10 +00001235 }
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001236
1237 // Update receive statistics. We count all layers, thus when you use layers
1238 // adding all key and delta frames might differ from frame count.
stefan@webrtc.org4cf1a8a2013-06-27 15:20:14 +00001239 if (frame.IsSessionComplete()) {
pbos@webrtc.orgce4e9a32014-12-18 13:50:16 +00001240 if (frame.FrameType() == kVideoFrameKey) {
1241 ++receive_statistics_.key_frames;
1242 } else {
1243 ++receive_statistics_.delta_frames;
1244 }
pbos@webrtc.org55707692014-12-19 15:45:03 +00001245 if (stats_callback_ != NULL)
1246 stats_callback_->OnFrameCountsUpdated(receive_statistics_);
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001247 }
niklase@google.com470e71d2011-07-07 08:21:25 +00001248}
1249
agalusza@google.comd818dcb2013-07-29 21:48:11 +00001250void VCMJitterBuffer::UpdateAveragePacketsPerFrame(int current_number_packets) {
1251 if (frame_counter_ > kFastConvergeThreshold) {
1252 average_packets_per_frame_ = average_packets_per_frame_
1253 * (1 - kNormalConvergeMultiplier)
1254 + current_number_packets * kNormalConvergeMultiplier;
1255 } else if (frame_counter_ > 0) {
1256 average_packets_per_frame_ = average_packets_per_frame_
1257 * (1 - kFastConvergeMultiplier)
1258 + current_number_packets * kFastConvergeMultiplier;
1259 frame_counter_++;
1260 } else {
1261 average_packets_per_frame_ = current_number_packets;
1262 frame_counter_++;
1263 }
1264}
1265
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001266// Must be called under the critical section |crit_sect_|.
stefan@webrtc.orga64300a2013-03-04 15:24:40 +00001267void VCMJitterBuffer::CleanUpOldOrEmptyFrames() {
pbos@webrtc.org4f16c872014-11-24 09:06:48 +00001268 decodable_frames_.CleanUpOldOrEmptyFrames(&last_decoded_state_,
1269 &free_frames_);
1270 incomplete_frames_.CleanUpOldOrEmptyFrames(&last_decoded_state_,
1271 &free_frames_);
stefan@webrtc.orga64300a2013-03-04 15:24:40 +00001272 if (!last_decoded_state_.in_initial_state()) {
1273 DropPacketsFromNackList(last_decoded_state_.sequence_num());
1274 }
mikhal@webrtc.org832caca2011-12-13 21:15:05 +00001275}
1276
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001277// Must be called from within |crit_sect_|.
1278bool VCMJitterBuffer::IsPacketRetransmitted(const VCMPacket& packet) const {
stefan@webrtc.orga64300a2013-03-04 15:24:40 +00001279 return missing_sequence_numbers_.find(packet.seqNum) !=
1280 missing_sequence_numbers_.end();
niklase@google.com470e71d2011-07-07 08:21:25 +00001281}
1282
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001283// Must be called under the critical section |crit_sect_|. Should never be
1284// called with retransmitted frames, they must be filtered out before this
1285// function is called.
1286void VCMJitterBuffer::UpdateJitterEstimate(const VCMJitterSample& sample,
1287 bool incomplete_frame) {
1288 if (sample.latest_packet_time == -1) {
1289 return;
1290 }
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001291 UpdateJitterEstimate(sample.latest_packet_time, sample.timestamp,
1292 sample.frame_size, incomplete_frame);
1293}
1294
1295// Must be called under the critical section crit_sect_. Should never be
1296// called with retransmitted frames, they must be filtered out before this
1297// function is called.
1298void VCMJitterBuffer::UpdateJitterEstimate(const VCMFrameBuffer& frame,
1299 bool incomplete_frame) {
1300 if (frame.LatestPacketTimeMs() == -1) {
1301 return;
1302 }
1303 // No retransmitted frames should be a part of the jitter
1304 // estimate.
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001305 UpdateJitterEstimate(frame.LatestPacketTimeMs(), frame.TimeStamp(),
1306 frame.Length(), incomplete_frame);
1307}
1308
1309// Must be called under the critical section |crit_sect_|. Should never be
1310// called with retransmitted frames, they must be filtered out before this
1311// function is called.
1312void VCMJitterBuffer::UpdateJitterEstimate(
1313 int64_t latest_packet_time_ms,
1314 uint32_t timestamp,
1315 unsigned int frame_size,
1316 bool incomplete_frame) {
1317 if (latest_packet_time_ms == -1) {
1318 return;
1319 }
1320 int64_t frame_delay;
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001321 bool not_reordered = inter_frame_delay_.CalculateDelay(timestamp,
1322 &frame_delay,
1323 latest_packet_time_ms);
1324 // Filter out frames which have been reordered in time by the network
1325 if (not_reordered) {
1326 // Update the jitter estimate with the new samples
1327 jitter_estimate_.UpdateEstimate(frame_delay, frame_size, incomplete_frame);
1328 }
1329}
1330
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001331bool VCMJitterBuffer::WaitForRetransmissions() {
1332 if (nack_mode_ == kNoNack) {
1333 // NACK disabled -> don't wait for retransmissions.
1334 return false;
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001335 }
stefan@webrtc.orga64300a2013-03-04 15:24:40 +00001336 // Evaluate if the RTT is higher than |high_rtt_nack_threshold_ms_|, and in
1337 // that case we don't wait for retransmissions.
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001338 if (high_rtt_nack_threshold_ms_ >= 0 &&
pkasting@chromium.org16825b12015-01-12 21:51:21 +00001339 rtt_ms_ >= high_rtt_nack_threshold_ms_) {
stefan@webrtc.org912981f2012-10-12 07:04:52 +00001340 return false;
1341 }
1342 return true;
1343}
stefan@webrtc.org932ab182011-11-29 11:33:31 +00001344} // namespace webrtc