blob: db09e7612dedd63b59d18f0414bf2308608e1eb1 [file] [log] [blame]
philipelbe7a9e52016-05-19 12:19:35 +02001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "modules/video_coding/frame_buffer2.h"
philipelbe7a9e52016-05-19 12:19:35 +020012
13#include <algorithm>
philipele0b2f152016-09-28 10:23:49 +020014#include <cstring>
15#include <queue>
philipelbe7a9e52016-05-19 12:19:35 +020016
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020017#include "modules/video_coding/include/video_coding_defines.h"
18#include "modules/video_coding/jitter_estimator.h"
19#include "modules/video_coding/timing.h"
20#include "rtc_base/checks.h"
21#include "rtc_base/logging.h"
22#include "rtc_base/trace_event.h"
23#include "system_wrappers/include/clock.h"
philipel707f2782017-10-02 14:10:28 +020024#include "system_wrappers/include/field_trial.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020025#include "system_wrappers/include/metrics.h"
philipelbe7a9e52016-05-19 12:19:35 +020026
27namespace webrtc {
28namespace video_coding {
29
30namespace {
philipele0b2f152016-09-28 10:23:49 +020031// Max number of frames the buffer will hold.
32constexpr int kMaxFramesBuffered = 600;
philipelbe7a9e52016-05-19 12:19:35 +020033
philipele0b2f152016-09-28 10:23:49 +020034// Max number of decoded frame info that will be saved.
philipelfd5a20f2016-11-15 00:57:57 -080035constexpr int kMaxFramesHistory = 50;
philipel65e1f942017-07-24 08:26:53 -070036
37constexpr int64_t kLogNonDecodedIntervalMs = 5000;
philipelbe7a9e52016-05-19 12:19:35 +020038} // namespace
39
philipelbe7a9e52016-05-19 12:19:35 +020040FrameBuffer::FrameBuffer(Clock* clock,
41 VCMJitterEstimator* jitter_estimator,
philipela45102f2017-02-22 05:30:39 -080042 VCMTiming* timing,
43 VCMReceiveStatisticsCallback* stats_callback)
philipelbe7a9e52016-05-19 12:19:35 +020044 : clock_(clock),
tommi0a735642017-03-14 06:23:57 -070045 new_continuous_frame_event_(false, false),
philipelbe7a9e52016-05-19 12:19:35 +020046 jitter_estimator_(jitter_estimator),
47 timing_(timing),
philipel4f6cd6a2016-08-03 10:59:32 +020048 inter_frame_delay_(clock_->TimeInMilliseconds()),
gnishb2a318b2017-05-10 09:21:33 -070049 last_decoded_frame_timestamp_(0),
philipele0b2f152016-09-28 10:23:49 +020050 last_decoded_frame_it_(frames_.end()),
51 last_continuous_frame_it_(frames_.end()),
52 num_frames_history_(0),
53 num_frames_buffered_(0),
philipel29f730e2017-03-15 08:10:08 -070054 stopped_(false),
philipela45102f2017-02-22 05:30:39 -080055 protection_mode_(kProtectionNack),
philipel65e1f942017-07-24 08:26:53 -070056 stats_callback_(stats_callback),
57 last_log_non_decoded_ms_(-kLogNonDecodedIntervalMs) {}
philipel266f0a42016-11-28 08:49:07 -080058
philipela45102f2017-02-22 05:30:39 -080059FrameBuffer::~FrameBuffer() {}
philipelbe7a9e52016-05-19 12:19:35 +020060
philipel75562822016-09-05 10:57:41 +020061FrameBuffer::ReturnReason FrameBuffer::NextFrame(
62 int64_t max_wait_time_ms,
philipele7c891f2018-02-22 14:35:06 +010063 std::unique_ptr<EncodedFrame>* frame_out,
philipel3042c2d2017-08-18 04:55:02 -070064 bool keyframe_required) {
tommidb23ea62017-03-03 07:21:18 -080065 TRACE_EVENT0("webrtc", "FrameBuffer::NextFrame");
philipel1c056252017-01-31 09:53:12 -080066 int64_t latest_return_time_ms =
67 clock_->TimeInMilliseconds() + max_wait_time_ms;
philipel504c47d2016-06-30 17:33:02 +020068 int64_t wait_ms = max_wait_time_ms;
philipel29f730e2017-03-15 08:10:08 -070069 int64_t now_ms = 0;
philipele0b2f152016-09-28 10:23:49 +020070
71 do {
philipel29f730e2017-03-15 08:10:08 -070072 now_ms = clock_->TimeInMilliseconds();
philipel504c47d2016-06-30 17:33:02 +020073 {
74 rtc::CritScope lock(&crit_);
tommi0a735642017-03-14 06:23:57 -070075 new_continuous_frame_event_.Reset();
philipel29f730e2017-03-15 08:10:08 -070076 if (stopped_)
77 return kStopped;
78
79 wait_ms = max_wait_time_ms;
80
philipele0b2f152016-09-28 10:23:49 +020081 // Need to hold |crit_| in order to use |frames_|, therefore we
82 // set it here in the loop instead of outside the loop in order to not
83 // acquire the lock unnecesserily.
philipel1c056252017-01-31 09:53:12 -080084 next_frame_it_ = frames_.end();
philipelbe7a9e52016-05-19 12:19:35 +020085
philipele0b2f152016-09-28 10:23:49 +020086 // |frame_it| points to the first frame after the
87 // |last_decoded_frame_it_|.
88 auto frame_it = frames_.end();
89 if (last_decoded_frame_it_ == frames_.end()) {
90 frame_it = frames_.begin();
philipelbe7a9e52016-05-19 12:19:35 +020091 } else {
philipele0b2f152016-09-28 10:23:49 +020092 frame_it = last_decoded_frame_it_;
93 ++frame_it;
philipelbe7a9e52016-05-19 12:19:35 +020094 }
philipele0b2f152016-09-28 10:23:49 +020095
96 // |continuous_end_it| points to the first frame after the
97 // |last_continuous_frame_it_|.
98 auto continuous_end_it = last_continuous_frame_it_;
99 if (continuous_end_it != frames_.end())
100 ++continuous_end_it;
101
philipel146a48b2017-04-20 04:04:38 -0700102 for (; frame_it != continuous_end_it && frame_it != frames_.end();
103 ++frame_it) {
philipel93e451b2016-10-06 12:25:13 +0200104 if (!frame_it->second.continuous ||
105 frame_it->second.num_missing_decodable > 0) {
philipele0b2f152016-09-28 10:23:49 +0200106 continue;
philipel93e451b2016-10-06 12:25:13 +0200107 }
philipele0b2f152016-09-28 10:23:49 +0200108
philipele7c891f2018-02-22 14:35:06 +0100109 EncodedFrame* frame = frame_it->second.frame.get();
philipel3042c2d2017-08-18 04:55:02 -0700110
111 if (keyframe_required && !frame->is_keyframe())
112 continue;
113
philipel1c056252017-01-31 09:53:12 -0800114 next_frame_it_ = frame_it;
philipele0b2f152016-09-28 10:23:49 +0200115 if (frame->RenderTime() == -1)
116 frame->SetRenderTime(timing_->RenderTimeMs(frame->timestamp, now_ms));
117 wait_ms = timing_->MaxWaitingTime(frame->RenderTime(), now_ms);
118
119 // This will cause the frame buffer to prefer high framerate rather
120 // than high resolution in the case of the decoder not decoding fast
121 // enough and the stream has multiple spatial and temporal layers.
122 if (wait_ms == 0)
123 continue;
124
125 break;
126 }
127 } // rtc::Critscope lock(&crit_);
128
philipel1c056252017-01-31 09:53:12 -0800129 wait_ms = std::min<int64_t>(wait_ms, latest_return_time_ms - now_ms);
philipele0b2f152016-09-28 10:23:49 +0200130 wait_ms = std::max<int64_t>(wait_ms, 0);
tommi0a735642017-03-14 06:23:57 -0700131 } while (new_continuous_frame_event_.Wait(wait_ms));
philipele0b2f152016-09-28 10:23:49 +0200132
philipel29f730e2017-03-15 08:10:08 -0700133 {
134 rtc::CritScope lock(&crit_);
135 now_ms = clock_->TimeInMilliseconds();
136 if (next_frame_it_ != frames_.end()) {
philipele7c891f2018-02-22 14:35:06 +0100137 std::unique_ptr<EncodedFrame> frame =
philipel29f730e2017-03-15 08:10:08 -0700138 std::move(next_frame_it_->second.frame);
philipele0b2f152016-09-28 10:23:49 +0200139
philipel29f730e2017-03-15 08:10:08 -0700140 if (!frame->delayed_by_retransmission()) {
141 int64_t frame_delay;
philipele0754302017-01-25 08:56:23 -0800142
philipel29f730e2017-03-15 08:10:08 -0700143 if (inter_frame_delay_.CalculateDelay(frame->timestamp, &frame_delay,
144 frame->ReceivedTime())) {
145 jitter_estimator_->UpdateEstimate(frame_delay, frame->size());
146 }
147
148 float rtt_mult = protection_mode_ == kProtectionNackFEC ? 0.0 : 1.0;
149 timing_->SetJitterDelay(jitter_estimator_->GetJitterEstimate(rtt_mult));
150 timing_->UpdateCurrentDelay(frame->RenderTime(), now_ms);
philipele21be1d2017-09-25 06:37:12 -0700151 } else {
philipel707f2782017-10-02 14:10:28 +0200152 if (webrtc::field_trial::IsEnabled("WebRTC-AddRttToPlayoutDelay"))
153 jitter_estimator_->FrameNacked();
philipele0754302017-01-25 08:56:23 -0800154 }
155
stefan95e97542017-05-23 09:52:18 -0700156 // Gracefully handle bad RTP timestamps and render time issues.
157 if (HasBadRenderTiming(*frame, now_ms)) {
158 jitter_estimator_->Reset();
159 timing_->Reset();
160 frame->SetRenderTime(timing_->RenderTimeMs(frame->timestamp, now_ms));
161 }
162
philipel29f730e2017-03-15 08:10:08 -0700163 UpdateJitterDelay();
ilnik2edc6842017-07-06 03:06:50 -0700164 UpdateTimingFrameInfo();
philipel29f730e2017-03-15 08:10:08 -0700165 PropagateDecodability(next_frame_it_->second);
brandtr9078d8c2017-04-27 07:07:27 -0700166
167 // Sanity check for RTP timestamp monotonicity.
168 if (last_decoded_frame_it_ != frames_.end()) {
169 const FrameKey& last_decoded_frame_key = last_decoded_frame_it_->first;
170 const FrameKey& frame_key = next_frame_it_->first;
171
172 const bool frame_is_higher_spatial_layer_of_last_decoded_frame =
173 last_decoded_frame_timestamp_ == frame->timestamp &&
174 last_decoded_frame_key.picture_id == frame_key.picture_id &&
175 last_decoded_frame_key.spatial_layer < frame_key.spatial_layer;
176
177 if (AheadOrAt(last_decoded_frame_timestamp_, frame->timestamp) &&
178 !frame_is_higher_spatial_layer_of_last_decoded_frame) {
179 // TODO(brandtr): Consider clearing the entire buffer when we hit
180 // these conditions.
Mirko Bonadei675513b2017-11-09 11:09:25 +0100181 RTC_LOG(LS_WARNING)
182 << "Frame with (timestamp:picture_id:spatial_id) ("
183 << frame->timestamp << ":" << frame->picture_id << ":"
184 << static_cast<int>(frame->spatial_layer) << ")"
185 << " sent to decoder after frame with"
186 << " (timestamp:picture_id:spatial_id) ("
187 << last_decoded_frame_timestamp_ << ":"
188 << last_decoded_frame_key.picture_id << ":"
189 << static_cast<int>(last_decoded_frame_key.spatial_layer) << ").";
brandtr9078d8c2017-04-27 07:07:27 -0700190 }
191 }
192
philipel29f730e2017-03-15 08:10:08 -0700193 AdvanceLastDecodedFrame(next_frame_it_);
194 last_decoded_frame_timestamp_ = frame->timestamp;
195 *frame_out = std::move(frame);
196 return kFrameFound;
philipelbe7a9e52016-05-19 12:19:35 +0200197 }
tommi0a735642017-03-14 06:23:57 -0700198 }
199
200 if (latest_return_time_ms - now_ms > 0) {
philipel1c056252017-01-31 09:53:12 -0800201 // If |next_frame_it_ == frames_.end()| and there is still time left, it
202 // means that the frame buffer was cleared as the thread in this function
203 // was waiting to acquire |crit_| in order to return. Wait for the
204 // remaining time and then return.
205 return NextFrame(latest_return_time_ms - now_ms, frame_out);
philipelbe7a9e52016-05-19 12:19:35 +0200206 }
tommi0a735642017-03-14 06:23:57 -0700207
208 return kTimeout;
philipelbe7a9e52016-05-19 12:19:35 +0200209}
210
philipele7c891f2018-02-22 14:35:06 +0100211bool FrameBuffer::HasBadRenderTiming(const EncodedFrame& frame,
212 int64_t now_ms) {
stefan95e97542017-05-23 09:52:18 -0700213 // Assume that render timing errors are due to changes in the video stream.
214 int64_t render_time_ms = frame.RenderTimeMs();
215 const int64_t kMaxVideoDelayMs = 10000;
216 if (render_time_ms < 0) {
217 return true;
218 }
219 if (std::abs(render_time_ms - now_ms) > kMaxVideoDelayMs) {
220 int frame_delay = static_cast<int>(std::abs(render_time_ms - now_ms));
Mirko Bonadei675513b2017-11-09 11:09:25 +0100221 RTC_LOG(LS_WARNING)
222 << "A frame about to be decoded is out of the configured "
223 << "delay bounds (" << frame_delay << " > " << kMaxVideoDelayMs
224 << "). Resetting the video jitter buffer.";
stefan95e97542017-05-23 09:52:18 -0700225 return true;
226 }
227 if (static_cast<int>(timing_->TargetVideoDelay()) > kMaxVideoDelayMs) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100228 RTC_LOG(LS_WARNING) << "The video target delay has grown larger than "
229 << kMaxVideoDelayMs << " ms.";
stefan95e97542017-05-23 09:52:18 -0700230 return true;
231 }
232 return false;
233}
234
philipel4f6cd6a2016-08-03 10:59:32 +0200235void FrameBuffer::SetProtectionMode(VCMVideoProtection mode) {
tommidb23ea62017-03-03 07:21:18 -0800236 TRACE_EVENT0("webrtc", "FrameBuffer::SetProtectionMode");
philipel4f6cd6a2016-08-03 10:59:32 +0200237 rtc::CritScope lock(&crit_);
238 protection_mode_ = mode;
239}
240
philipel504c47d2016-06-30 17:33:02 +0200241void FrameBuffer::Start() {
tommidb23ea62017-03-03 07:21:18 -0800242 TRACE_EVENT0("webrtc", "FrameBuffer::Start");
philipel29f730e2017-03-15 08:10:08 -0700243 rtc::CritScope lock(&crit_);
244 stopped_ = false;
philipel504c47d2016-06-30 17:33:02 +0200245}
246
247void FrameBuffer::Stop() {
tommidb23ea62017-03-03 07:21:18 -0800248 TRACE_EVENT0("webrtc", "FrameBuffer::Stop");
philipel29f730e2017-03-15 08:10:08 -0700249 rtc::CritScope lock(&crit_);
250 stopped_ = true;
tommi0a735642017-03-14 06:23:57 -0700251 new_continuous_frame_event_.Set();
philipel504c47d2016-06-30 17:33:02 +0200252}
253
philipele21be1d2017-09-25 06:37:12 -0700254void FrameBuffer::UpdateRtt(int64_t rtt_ms) {
255 rtc::CritScope lock(&crit_);
256 jitter_estimator_->UpdateRtt(rtt_ms);
257}
258
philipele7c891f2018-02-22 14:35:06 +0100259bool FrameBuffer::ValidReferences(const EncodedFrame& frame) const {
philipel3b3c9c42017-09-11 09:38:36 -0700260 if (frame.picture_id < 0)
261 return false;
262
philipel112adf92017-06-15 09:06:21 -0700263 for (size_t i = 0; i < frame.num_references; ++i) {
philipel3b3c9c42017-09-11 09:38:36 -0700264 if (frame.references[i] < 0 || frame.references[i] >= frame.picture_id)
philipel112adf92017-06-15 09:06:21 -0700265 return false;
philipel3b3c9c42017-09-11 09:38:36 -0700266
philipel112adf92017-06-15 09:06:21 -0700267 for (size_t j = i + 1; j < frame.num_references; ++j) {
268 if (frame.references[i] == frame.references[j])
269 return false;
270 }
271 }
272
273 if (frame.inter_layer_predicted && frame.spatial_layer == 0)
274 return false;
275
276 return true;
277}
278
philipele7c891f2018-02-22 14:35:06 +0100279void FrameBuffer::UpdatePlayoutDelays(const EncodedFrame& frame) {
gnishb2a318b2017-05-10 09:21:33 -0700280 TRACE_EVENT0("webrtc", "FrameBuffer::UpdatePlayoutDelays");
281 PlayoutDelay playout_delay = frame.EncodedImage().playout_delay_;
282 if (playout_delay.min_ms >= 0)
283 timing_->set_min_playout_delay(playout_delay.min_ms);
284
285 if (playout_delay.max_ms >= 0)
286 timing_->set_max_playout_delay(playout_delay.max_ms);
287}
288
philipele7c891f2018-02-22 14:35:06 +0100289int64_t FrameBuffer::InsertFrame(std::unique_ptr<EncodedFrame> frame) {
tommidb23ea62017-03-03 07:21:18 -0800290 TRACE_EVENT0("webrtc", "FrameBuffer::InsertFrame");
philipel93e451b2016-10-06 12:25:13 +0200291 RTC_DCHECK(frame);
philipela45102f2017-02-22 05:30:39 -0800292 if (stats_callback_)
ilnik6d5b4d62017-08-30 03:32:14 -0700293 stats_callback_->OnCompleteFrame(frame->is_keyframe(), frame->size(),
294 frame->contentType());
philipelbe7a9e52016-05-19 12:19:35 +0200295 FrameKey key(frame->picture_id, frame->spatial_layer);
tommi0a735642017-03-14 06:23:57 -0700296
297 rtc::CritScope lock(&crit_);
philipel29f730e2017-03-15 08:10:08 -0700298
philipel1610f942017-12-12 13:58:31 +0100299 int64_t last_continuous_picture_id =
philipele0b2f152016-09-28 10:23:49 +0200300 last_continuous_frame_it_ == frames_.end()
301 ? -1
302 : last_continuous_frame_it_->first.picture_id;
303
philipel112adf92017-06-15 09:06:21 -0700304 if (!ValidReferences(*frame)) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100305 RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) ("
306 << key.picture_id << ":"
307 << static_cast<int>(key.spatial_layer)
308 << ") has invalid frame references, dropping frame.";
philipel112adf92017-06-15 09:06:21 -0700309 return last_continuous_picture_id;
310 }
311
philipele0b2f152016-09-28 10:23:49 +0200312 if (num_frames_buffered_ >= kMaxFramesBuffered) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100313 RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) ("
314 << key.picture_id << ":"
315 << static_cast<int>(key.spatial_layer)
316 << ") could not be inserted due to the frame "
317 << "buffer being full, dropping frame.";
philipele0b2f152016-09-28 10:23:49 +0200318 return last_continuous_picture_id;
319 }
320
philipele0b2f152016-09-28 10:23:49 +0200321 if (last_decoded_frame_it_ != frames_.end() &&
philipelf6842692017-04-28 03:29:15 -0700322 key <= last_decoded_frame_it_->first) {
philipelfcc60062017-01-18 05:35:20 -0800323 if (AheadOf(frame->timestamp, last_decoded_frame_timestamp_) &&
philipel3042c2d2017-08-18 04:55:02 -0700324 frame->is_keyframe()) {
philipelfcc60062017-01-18 05:35:20 -0800325 // If this frame has a newer timestamp but an earlier picture id then we
326 // assume there has been a jump in the picture id due to some encoder
327 // reconfiguration or some other reason. Even though this is not according
328 // to spec we can still continue to decode from this frame if it is a
329 // keyframe.
Mirko Bonadei675513b2017-11-09 11:09:25 +0100330 RTC_LOG(LS_WARNING)
331 << "A jump in picture id was detected, clearing buffer.";
philipelfcc60062017-01-18 05:35:20 -0800332 ClearFramesAndHistory();
333 last_continuous_picture_id = -1;
334 } else {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100335 RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) ("
336 << key.picture_id << ":"
337 << static_cast<int>(key.spatial_layer)
338 << ") inserted after frame ("
339 << last_decoded_frame_it_->first.picture_id << ":"
340 << static_cast<int>(
341 last_decoded_frame_it_->first.spatial_layer)
342 << ") was handed off for decoding, dropping frame.";
philipelfcc60062017-01-18 05:35:20 -0800343 return last_continuous_picture_id;
344 }
philipele0b2f152016-09-28 10:23:49 +0200345 }
346
philipel146a48b2017-04-20 04:04:38 -0700347 // Test if inserting this frame would cause the order of the frames to become
348 // ambiguous (covering more than half the interval of 2^16). This can happen
349 // when the picture id make large jumps mid stream.
350 if (!frames_.empty() &&
351 key < frames_.begin()->first &&
352 frames_.rbegin()->first < key) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100353 RTC_LOG(LS_WARNING)
354 << "A jump in picture id was detected, clearing buffer.";
philipel146a48b2017-04-20 04:04:38 -0700355 ClearFramesAndHistory();
356 last_continuous_picture_id = -1;
357 }
358
philipele0b2f152016-09-28 10:23:49 +0200359 auto info = frames_.insert(std::make_pair(key, FrameInfo())).first;
360
philipel93e451b2016-10-06 12:25:13 +0200361 if (info->second.frame) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100362 RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) ("
363 << key.picture_id << ":"
364 << static_cast<int>(key.spatial_layer)
365 << ") already inserted, dropping frame.";
philipele0b2f152016-09-28 10:23:49 +0200366 return last_continuous_picture_id;
367 }
368
philipel93e451b2016-10-06 12:25:13 +0200369 if (!UpdateFrameInfoWithIncomingFrame(*frame, info))
370 return last_continuous_picture_id;
gnishb2a318b2017-05-10 09:21:33 -0700371 UpdatePlayoutDelays(*frame);
philipele0b2f152016-09-28 10:23:49 +0200372 info->second.frame = std::move(frame);
373 ++num_frames_buffered_;
374
375 if (info->second.num_missing_continuous == 0) {
376 info->second.continuous = true;
377 PropagateContinuity(info);
378 last_continuous_picture_id = last_continuous_frame_it_->first.picture_id;
379
380 // Since we now have new continuous frames there might be a better frame
381 // to return from NextFrame. Signal that thread so that it again can choose
382 // which frame to return.
tommi0a735642017-03-14 06:23:57 -0700383 new_continuous_frame_event_.Set();
philipele0b2f152016-09-28 10:23:49 +0200384 }
385
386 return last_continuous_picture_id;
philipelbe7a9e52016-05-19 12:19:35 +0200387}
388
philipele0b2f152016-09-28 10:23:49 +0200389void FrameBuffer::PropagateContinuity(FrameMap::iterator start) {
tommidb23ea62017-03-03 07:21:18 -0800390 TRACE_EVENT0("webrtc", "FrameBuffer::PropagateContinuity");
philipele0b2f152016-09-28 10:23:49 +0200391 RTC_DCHECK(start->second.continuous);
392 if (last_continuous_frame_it_ == frames_.end())
393 last_continuous_frame_it_ = start;
394
395 std::queue<FrameMap::iterator> continuous_frames;
396 continuous_frames.push(start);
397
398 // A simple BFS to traverse continuous frames.
399 while (!continuous_frames.empty()) {
400 auto frame = continuous_frames.front();
401 continuous_frames.pop();
402
403 if (last_continuous_frame_it_->first < frame->first)
404 last_continuous_frame_it_ = frame;
405
406 // Loop through all dependent frames, and if that frame no longer has
407 // any unfulfilled dependencies then that frame is continuous as well.
408 for (size_t d = 0; d < frame->second.num_dependent_frames; ++d) {
409 auto frame_ref = frames_.find(frame->second.dependent_frames[d]);
philipel112adf92017-06-15 09:06:21 -0700410 RTC_DCHECK(frame_ref != frames_.end());
philipele0b2f152016-09-28 10:23:49 +0200411
philipel112adf92017-06-15 09:06:21 -0700412 // TODO(philipel): Look into why we've seen this happen.
413 if (frame_ref != frames_.end()) {
414 --frame_ref->second.num_missing_continuous;
415 if (frame_ref->second.num_missing_continuous == 0) {
416 frame_ref->second.continuous = true;
417 continuous_frames.push(frame_ref);
418 }
philipele0b2f152016-09-28 10:23:49 +0200419 }
420 }
421 }
422}
423
424void FrameBuffer::PropagateDecodability(const FrameInfo& info) {
tommidb23ea62017-03-03 07:21:18 -0800425 TRACE_EVENT0("webrtc", "FrameBuffer::PropagateDecodability");
tommie95b78b2017-05-14 07:23:11 -0700426 RTC_CHECK(info.num_dependent_frames < FrameInfo::kMaxNumDependentFrames);
philipele0b2f152016-09-28 10:23:49 +0200427 for (size_t d = 0; d < info.num_dependent_frames; ++d) {
428 auto ref_info = frames_.find(info.dependent_frames[d]);
philipel93e451b2016-10-06 12:25:13 +0200429 RTC_DCHECK(ref_info != frames_.end());
tommie95b78b2017-05-14 07:23:11 -0700430 // TODO(philipel): Look into why we've seen this happen.
431 if (ref_info != frames_.end()) {
432 RTC_DCHECK_GT(ref_info->second.num_missing_decodable, 0U);
433 --ref_info->second.num_missing_decodable;
434 }
philipele0b2f152016-09-28 10:23:49 +0200435 }
436}
437
438void FrameBuffer::AdvanceLastDecodedFrame(FrameMap::iterator decoded) {
tommidb23ea62017-03-03 07:21:18 -0800439 TRACE_EVENT0("webrtc", "FrameBuffer::AdvanceLastDecodedFrame");
philipele0b2f152016-09-28 10:23:49 +0200440 if (last_decoded_frame_it_ == frames_.end()) {
441 last_decoded_frame_it_ = frames_.begin();
442 } else {
443 RTC_DCHECK(last_decoded_frame_it_->first < decoded->first);
444 ++last_decoded_frame_it_;
445 }
446 --num_frames_buffered_;
447 ++num_frames_history_;
448
449 // First, delete non-decoded frames from the history.
450 while (last_decoded_frame_it_ != decoded) {
451 if (last_decoded_frame_it_->second.frame)
452 --num_frames_buffered_;
453 last_decoded_frame_it_ = frames_.erase(last_decoded_frame_it_);
philipelbe7a9e52016-05-19 12:19:35 +0200454 }
455
philipele0b2f152016-09-28 10:23:49 +0200456 // Then remove old history if we have too much history saved.
457 if (num_frames_history_ > kMaxFramesHistory) {
458 frames_.erase(frames_.begin());
459 --num_frames_history_;
460 }
461}
462
philipele7c891f2018-02-22 14:35:06 +0100463bool FrameBuffer::UpdateFrameInfoWithIncomingFrame(const EncodedFrame& frame,
philipele0b2f152016-09-28 10:23:49 +0200464 FrameMap::iterator info) {
tommidb23ea62017-03-03 07:21:18 -0800465 TRACE_EVENT0("webrtc", "FrameBuffer::UpdateFrameInfoWithIncomingFrame");
philipele0b2f152016-09-28 10:23:49 +0200466 FrameKey key(frame.picture_id, frame.spatial_layer);
467 info->second.num_missing_continuous = frame.num_references;
468 info->second.num_missing_decodable = frame.num_references;
469
470 RTC_DCHECK(last_decoded_frame_it_ == frames_.end() ||
471 last_decoded_frame_it_->first < info->first);
472
473 // Check how many dependencies that have already been fulfilled.
474 for (size_t i = 0; i < frame.num_references; ++i) {
475 FrameKey ref_key(frame.references[i], frame.spatial_layer);
476 auto ref_info = frames_.find(ref_key);
477
478 // Does |frame| depend on a frame earlier than the last decoded frame?
479 if (last_decoded_frame_it_ != frames_.end() &&
480 ref_key <= last_decoded_frame_it_->first) {
481 if (ref_info == frames_.end()) {
philipel65e1f942017-07-24 08:26:53 -0700482 int64_t now_ms = clock_->TimeInMilliseconds();
483 if (last_log_non_decoded_ms_ + kLogNonDecodedIntervalMs < now_ms) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100484 RTC_LOG(LS_WARNING)
philipel65e1f942017-07-24 08:26:53 -0700485 << "Frame with (picture_id:spatial_id) (" << key.picture_id << ":"
486 << static_cast<int>(key.spatial_layer)
487 << ") depends on a non-decoded frame more previous than"
488 << " the last decoded frame, dropping frame.";
489 last_log_non_decoded_ms_ = now_ms;
490 }
philipele0b2f152016-09-28 10:23:49 +0200491 return false;
492 }
493
494 --info->second.num_missing_continuous;
495 --info->second.num_missing_decodable;
496 } else {
497 if (ref_info == frames_.end())
498 ref_info = frames_.insert(std::make_pair(ref_key, FrameInfo())).first;
499
500 if (ref_info->second.continuous)
501 --info->second.num_missing_continuous;
502
503 // Add backwards reference so |frame| can be updated when new
504 // frames are inserted or decoded.
505 ref_info->second.dependent_frames[ref_info->second.num_dependent_frames] =
506 key;
tommie95b78b2017-05-14 07:23:11 -0700507 RTC_DCHECK_LT(ref_info->second.num_dependent_frames,
508 (FrameInfo::kMaxNumDependentFrames - 1));
509 // TODO(philipel): Look into why this could happen and handle
510 // appropriately.
511 if (ref_info->second.num_dependent_frames <
512 (FrameInfo::kMaxNumDependentFrames - 1)) {
513 ++ref_info->second.num_dependent_frames;
514 }
philipele0b2f152016-09-28 10:23:49 +0200515 }
philipel93e451b2016-10-06 12:25:13 +0200516 RTC_DCHECK_LE(ref_info->second.num_missing_continuous,
517 ref_info->second.num_missing_decodable);
philipelbe7a9e52016-05-19 12:19:35 +0200518 }
519
philipele0b2f152016-09-28 10:23:49 +0200520 // Check if we have the lower spatial layer frame.
philipelbe7a9e52016-05-19 12:19:35 +0200521 if (frame.inter_layer_predicted) {
philipele0b2f152016-09-28 10:23:49 +0200522 ++info->second.num_missing_continuous;
523 ++info->second.num_missing_decodable;
524
philipelbe7a9e52016-05-19 12:19:35 +0200525 FrameKey ref_key(frame.picture_id, frame.spatial_layer - 1);
philipele0b2f152016-09-28 10:23:49 +0200526 // Gets or create the FrameInfo for the referenced frame.
527 auto ref_info = frames_.insert(std::make_pair(ref_key, FrameInfo())).first;
528 if (ref_info->second.continuous)
529 --info->second.num_missing_continuous;
530
531 if (ref_info == last_decoded_frame_it_) {
532 --info->second.num_missing_decodable;
533 } else {
534 ref_info->second.dependent_frames[ref_info->second.num_dependent_frames] =
535 key;
536 ++ref_info->second.num_dependent_frames;
537 }
philipel93e451b2016-10-06 12:25:13 +0200538 RTC_DCHECK_LE(ref_info->second.num_missing_continuous,
539 ref_info->second.num_missing_decodable);
philipelbe7a9e52016-05-19 12:19:35 +0200540 }
541
philipel93e451b2016-10-06 12:25:13 +0200542 RTC_DCHECK_LE(info->second.num_missing_continuous,
543 info->second.num_missing_decodable);
544
philipelbe7a9e52016-05-19 12:19:35 +0200545 return true;
546}
547
philipelbe742702016-11-30 01:31:40 -0800548void FrameBuffer::UpdateJitterDelay() {
tommidb23ea62017-03-03 07:21:18 -0800549 TRACE_EVENT0("webrtc", "FrameBuffer::UpdateJitterDelay");
philipela45102f2017-02-22 05:30:39 -0800550 if (!stats_callback_)
551 return;
philipelbe742702016-11-30 01:31:40 -0800552
philipela45102f2017-02-22 05:30:39 -0800553 int decode_ms;
554 int max_decode_ms;
555 int current_delay_ms;
556 int target_delay_ms;
557 int jitter_buffer_ms;
558 int min_playout_delay_ms;
559 int render_delay_ms;
560 if (timing_->GetTimings(&decode_ms, &max_decode_ms, &current_delay_ms,
561 &target_delay_ms, &jitter_buffer_ms,
562 &min_playout_delay_ms, &render_delay_ms)) {
563 stats_callback_->OnFrameBufferTimingsUpdated(
564 decode_ms, max_decode_ms, current_delay_ms, target_delay_ms,
565 jitter_buffer_ms, min_playout_delay_ms, render_delay_ms);
philipelbe742702016-11-30 01:31:40 -0800566 }
philipel266f0a42016-11-28 08:49:07 -0800567}
568
ilnik2edc6842017-07-06 03:06:50 -0700569void FrameBuffer::UpdateTimingFrameInfo() {
570 TRACE_EVENT0("webrtc", "FrameBuffer::UpdateTimingFrameInfo");
571 rtc::Optional<TimingFrameInfo> info = timing_->GetTimingFrameInfo();
572 if (info)
573 stats_callback_->OnTimingFrameInfoUpdated(*info);
574}
575
philipelfcc60062017-01-18 05:35:20 -0800576void FrameBuffer::ClearFramesAndHistory() {
ilnik2edc6842017-07-06 03:06:50 -0700577 TRACE_EVENT0("webrtc", "FrameBuffer::ClearFramesAndHistory");
philipelfcc60062017-01-18 05:35:20 -0800578 frames_.clear();
579 last_decoded_frame_it_ = frames_.end();
580 last_continuous_frame_it_ = frames_.end();
philipel1c056252017-01-31 09:53:12 -0800581 next_frame_it_ = frames_.end();
philipelfcc60062017-01-18 05:35:20 -0800582 num_frames_history_ = 0;
583 num_frames_buffered_ = 0;
584}
585
philipelbe7a9e52016-05-19 12:19:35 +0200586} // namespace video_coding
587} // namespace webrtc