blob: 50c3763dc1ae687c82713b293ae51c49a07f155b [file] [log] [blame]
philipelbe7a9e52016-05-19 12:19:35 +02001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "modules/video_coding/frame_buffer2.h"
philipelbe7a9e52016-05-19 12:19:35 +020012
13#include <algorithm>
philipele0b2f152016-09-28 10:23:49 +020014#include <cstring>
15#include <queue>
philipelbe7a9e52016-05-19 12:19:35 +020016
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020017#include "modules/video_coding/include/video_coding_defines.h"
18#include "modules/video_coding/jitter_estimator.h"
19#include "modules/video_coding/timing.h"
20#include "rtc_base/checks.h"
21#include "rtc_base/logging.h"
22#include "rtc_base/trace_event.h"
23#include "system_wrappers/include/clock.h"
24#include "system_wrappers/include/metrics.h"
philipelbe7a9e52016-05-19 12:19:35 +020025
26namespace webrtc {
27namespace video_coding {
28
29namespace {
philipele0b2f152016-09-28 10:23:49 +020030// Max number of frames the buffer will hold.
31constexpr int kMaxFramesBuffered = 600;
philipelbe7a9e52016-05-19 12:19:35 +020032
philipele0b2f152016-09-28 10:23:49 +020033// Max number of decoded frame info that will be saved.
philipelfd5a20f2016-11-15 00:57:57 -080034constexpr int kMaxFramesHistory = 50;
philipel65e1f942017-07-24 08:26:53 -070035
36constexpr int64_t kLogNonDecodedIntervalMs = 5000;
philipelbe7a9e52016-05-19 12:19:35 +020037} // namespace
38
philipelbe7a9e52016-05-19 12:19:35 +020039FrameBuffer::FrameBuffer(Clock* clock,
40 VCMJitterEstimator* jitter_estimator,
philipela45102f2017-02-22 05:30:39 -080041 VCMTiming* timing,
42 VCMReceiveStatisticsCallback* stats_callback)
philipelbe7a9e52016-05-19 12:19:35 +020043 : clock_(clock),
tommi0a735642017-03-14 06:23:57 -070044 new_continuous_frame_event_(false, false),
philipelbe7a9e52016-05-19 12:19:35 +020045 jitter_estimator_(jitter_estimator),
46 timing_(timing),
philipel4f6cd6a2016-08-03 10:59:32 +020047 inter_frame_delay_(clock_->TimeInMilliseconds()),
gnishb2a318b2017-05-10 09:21:33 -070048 last_decoded_frame_timestamp_(0),
philipele0b2f152016-09-28 10:23:49 +020049 last_decoded_frame_it_(frames_.end()),
50 last_continuous_frame_it_(frames_.end()),
51 num_frames_history_(0),
52 num_frames_buffered_(0),
philipel29f730e2017-03-15 08:10:08 -070053 stopped_(false),
philipela45102f2017-02-22 05:30:39 -080054 protection_mode_(kProtectionNack),
philipel65e1f942017-07-24 08:26:53 -070055 stats_callback_(stats_callback),
56 last_log_non_decoded_ms_(-kLogNonDecodedIntervalMs) {}
philipel266f0a42016-11-28 08:49:07 -080057
philipela45102f2017-02-22 05:30:39 -080058FrameBuffer::~FrameBuffer() {}
philipelbe7a9e52016-05-19 12:19:35 +020059
philipel75562822016-09-05 10:57:41 +020060FrameBuffer::ReturnReason FrameBuffer::NextFrame(
61 int64_t max_wait_time_ms,
philipel3042c2d2017-08-18 04:55:02 -070062 std::unique_ptr<FrameObject>* frame_out,
63 bool keyframe_required) {
tommidb23ea62017-03-03 07:21:18 -080064 TRACE_EVENT0("webrtc", "FrameBuffer::NextFrame");
philipel1c056252017-01-31 09:53:12 -080065 int64_t latest_return_time_ms =
66 clock_->TimeInMilliseconds() + max_wait_time_ms;
philipel504c47d2016-06-30 17:33:02 +020067 int64_t wait_ms = max_wait_time_ms;
philipel29f730e2017-03-15 08:10:08 -070068 int64_t now_ms = 0;
philipele0b2f152016-09-28 10:23:49 +020069
70 do {
philipel29f730e2017-03-15 08:10:08 -070071 now_ms = clock_->TimeInMilliseconds();
philipel504c47d2016-06-30 17:33:02 +020072 {
73 rtc::CritScope lock(&crit_);
tommi0a735642017-03-14 06:23:57 -070074 new_continuous_frame_event_.Reset();
philipel29f730e2017-03-15 08:10:08 -070075 if (stopped_)
76 return kStopped;
77
78 wait_ms = max_wait_time_ms;
79
philipele0b2f152016-09-28 10:23:49 +020080 // Need to hold |crit_| in order to use |frames_|, therefore we
81 // set it here in the loop instead of outside the loop in order to not
82 // acquire the lock unnecesserily.
philipel1c056252017-01-31 09:53:12 -080083 next_frame_it_ = frames_.end();
philipelbe7a9e52016-05-19 12:19:35 +020084
philipele0b2f152016-09-28 10:23:49 +020085 // |frame_it| points to the first frame after the
86 // |last_decoded_frame_it_|.
87 auto frame_it = frames_.end();
88 if (last_decoded_frame_it_ == frames_.end()) {
89 frame_it = frames_.begin();
philipelbe7a9e52016-05-19 12:19:35 +020090 } else {
philipele0b2f152016-09-28 10:23:49 +020091 frame_it = last_decoded_frame_it_;
92 ++frame_it;
philipelbe7a9e52016-05-19 12:19:35 +020093 }
philipele0b2f152016-09-28 10:23:49 +020094
95 // |continuous_end_it| points to the first frame after the
96 // |last_continuous_frame_it_|.
97 auto continuous_end_it = last_continuous_frame_it_;
98 if (continuous_end_it != frames_.end())
99 ++continuous_end_it;
100
philipel146a48b2017-04-20 04:04:38 -0700101 for (; frame_it != continuous_end_it && frame_it != frames_.end();
102 ++frame_it) {
philipel93e451b2016-10-06 12:25:13 +0200103 if (!frame_it->second.continuous ||
104 frame_it->second.num_missing_decodable > 0) {
philipele0b2f152016-09-28 10:23:49 +0200105 continue;
philipel93e451b2016-10-06 12:25:13 +0200106 }
philipele0b2f152016-09-28 10:23:49 +0200107
108 FrameObject* frame = frame_it->second.frame.get();
philipel3042c2d2017-08-18 04:55:02 -0700109
110 if (keyframe_required && !frame->is_keyframe())
111 continue;
112
philipel1c056252017-01-31 09:53:12 -0800113 next_frame_it_ = frame_it;
philipele0b2f152016-09-28 10:23:49 +0200114 if (frame->RenderTime() == -1)
115 frame->SetRenderTime(timing_->RenderTimeMs(frame->timestamp, now_ms));
116 wait_ms = timing_->MaxWaitingTime(frame->RenderTime(), now_ms);
117
118 // This will cause the frame buffer to prefer high framerate rather
119 // than high resolution in the case of the decoder not decoding fast
120 // enough and the stream has multiple spatial and temporal layers.
121 if (wait_ms == 0)
122 continue;
123
124 break;
125 }
126 } // rtc::Critscope lock(&crit_);
127
philipel1c056252017-01-31 09:53:12 -0800128 wait_ms = std::min<int64_t>(wait_ms, latest_return_time_ms - now_ms);
philipele0b2f152016-09-28 10:23:49 +0200129 wait_ms = std::max<int64_t>(wait_ms, 0);
tommi0a735642017-03-14 06:23:57 -0700130 } while (new_continuous_frame_event_.Wait(wait_ms));
philipele0b2f152016-09-28 10:23:49 +0200131
philipel29f730e2017-03-15 08:10:08 -0700132 {
133 rtc::CritScope lock(&crit_);
134 now_ms = clock_->TimeInMilliseconds();
135 if (next_frame_it_ != frames_.end()) {
136 std::unique_ptr<FrameObject> frame =
137 std::move(next_frame_it_->second.frame);
philipele0b2f152016-09-28 10:23:49 +0200138
philipel29f730e2017-03-15 08:10:08 -0700139 if (!frame->delayed_by_retransmission()) {
140 int64_t frame_delay;
philipele0754302017-01-25 08:56:23 -0800141
philipel29f730e2017-03-15 08:10:08 -0700142 if (inter_frame_delay_.CalculateDelay(frame->timestamp, &frame_delay,
143 frame->ReceivedTime())) {
144 jitter_estimator_->UpdateEstimate(frame_delay, frame->size());
145 }
146
147 float rtt_mult = protection_mode_ == kProtectionNackFEC ? 0.0 : 1.0;
148 timing_->SetJitterDelay(jitter_estimator_->GetJitterEstimate(rtt_mult));
149 timing_->UpdateCurrentDelay(frame->RenderTime(), now_ms);
philipele0754302017-01-25 08:56:23 -0800150 }
151
stefan95e97542017-05-23 09:52:18 -0700152 // Gracefully handle bad RTP timestamps and render time issues.
153 if (HasBadRenderTiming(*frame, now_ms)) {
154 jitter_estimator_->Reset();
155 timing_->Reset();
156 frame->SetRenderTime(timing_->RenderTimeMs(frame->timestamp, now_ms));
157 }
158
philipel29f730e2017-03-15 08:10:08 -0700159 UpdateJitterDelay();
ilnik2edc6842017-07-06 03:06:50 -0700160 UpdateTimingFrameInfo();
philipel29f730e2017-03-15 08:10:08 -0700161 PropagateDecodability(next_frame_it_->second);
brandtr9078d8c2017-04-27 07:07:27 -0700162
163 // Sanity check for RTP timestamp monotonicity.
164 if (last_decoded_frame_it_ != frames_.end()) {
165 const FrameKey& last_decoded_frame_key = last_decoded_frame_it_->first;
166 const FrameKey& frame_key = next_frame_it_->first;
167
168 const bool frame_is_higher_spatial_layer_of_last_decoded_frame =
169 last_decoded_frame_timestamp_ == frame->timestamp &&
170 last_decoded_frame_key.picture_id == frame_key.picture_id &&
171 last_decoded_frame_key.spatial_layer < frame_key.spatial_layer;
172
173 if (AheadOrAt(last_decoded_frame_timestamp_, frame->timestamp) &&
174 !frame_is_higher_spatial_layer_of_last_decoded_frame) {
175 // TODO(brandtr): Consider clearing the entire buffer when we hit
176 // these conditions.
177 LOG(LS_WARNING) << "Frame with (timestamp:picture_id:spatial_id) ("
178 << frame->timestamp << ":" << frame->picture_id << ":"
179 << static_cast<int>(frame->spatial_layer) << ")"
180 << " sent to decoder after frame with"
181 << " (timestamp:picture_id:spatial_id) ("
182 << last_decoded_frame_timestamp_ << ":"
183 << last_decoded_frame_key.picture_id << ":"
184 << static_cast<int>(
185 last_decoded_frame_key.spatial_layer)
186 << ").";
187 }
188 }
189
philipel29f730e2017-03-15 08:10:08 -0700190 AdvanceLastDecodedFrame(next_frame_it_);
191 last_decoded_frame_timestamp_ = frame->timestamp;
192 *frame_out = std::move(frame);
193 return kFrameFound;
philipelbe7a9e52016-05-19 12:19:35 +0200194 }
tommi0a735642017-03-14 06:23:57 -0700195 }
196
197 if (latest_return_time_ms - now_ms > 0) {
philipel1c056252017-01-31 09:53:12 -0800198 // If |next_frame_it_ == frames_.end()| and there is still time left, it
199 // means that the frame buffer was cleared as the thread in this function
200 // was waiting to acquire |crit_| in order to return. Wait for the
201 // remaining time and then return.
202 return NextFrame(latest_return_time_ms - now_ms, frame_out);
philipelbe7a9e52016-05-19 12:19:35 +0200203 }
tommi0a735642017-03-14 06:23:57 -0700204
205 return kTimeout;
philipelbe7a9e52016-05-19 12:19:35 +0200206}
207
stefan95e97542017-05-23 09:52:18 -0700208bool FrameBuffer::HasBadRenderTiming(const FrameObject& frame, int64_t now_ms) {
209 // Assume that render timing errors are due to changes in the video stream.
210 int64_t render_time_ms = frame.RenderTimeMs();
211 const int64_t kMaxVideoDelayMs = 10000;
212 if (render_time_ms < 0) {
213 return true;
214 }
215 if (std::abs(render_time_ms - now_ms) > kMaxVideoDelayMs) {
216 int frame_delay = static_cast<int>(std::abs(render_time_ms - now_ms));
217 LOG(LS_WARNING) << "A frame about to be decoded is out of the configured "
218 << "delay bounds (" << frame_delay << " > "
219 << kMaxVideoDelayMs
220 << "). Resetting the video jitter buffer.";
221 return true;
222 }
223 if (static_cast<int>(timing_->TargetVideoDelay()) > kMaxVideoDelayMs) {
224 LOG(LS_WARNING) << "The video target delay has grown larger than "
225 << kMaxVideoDelayMs << " ms.";
226 return true;
227 }
228 return false;
229}
230
philipel4f6cd6a2016-08-03 10:59:32 +0200231void FrameBuffer::SetProtectionMode(VCMVideoProtection mode) {
tommidb23ea62017-03-03 07:21:18 -0800232 TRACE_EVENT0("webrtc", "FrameBuffer::SetProtectionMode");
philipel4f6cd6a2016-08-03 10:59:32 +0200233 rtc::CritScope lock(&crit_);
234 protection_mode_ = mode;
235}
236
philipel504c47d2016-06-30 17:33:02 +0200237void FrameBuffer::Start() {
tommidb23ea62017-03-03 07:21:18 -0800238 TRACE_EVENT0("webrtc", "FrameBuffer::Start");
philipel29f730e2017-03-15 08:10:08 -0700239 rtc::CritScope lock(&crit_);
240 stopped_ = false;
philipel504c47d2016-06-30 17:33:02 +0200241}
242
243void FrameBuffer::Stop() {
tommidb23ea62017-03-03 07:21:18 -0800244 TRACE_EVENT0("webrtc", "FrameBuffer::Stop");
philipel29f730e2017-03-15 08:10:08 -0700245 rtc::CritScope lock(&crit_);
246 stopped_ = true;
tommi0a735642017-03-14 06:23:57 -0700247 new_continuous_frame_event_.Set();
philipel504c47d2016-06-30 17:33:02 +0200248}
249
philipel112adf92017-06-15 09:06:21 -0700250bool FrameBuffer::ValidReferences(const FrameObject& frame) const {
philipel3b3c9c42017-09-11 09:38:36 -0700251 if (frame.picture_id < 0)
252 return false;
253
philipel112adf92017-06-15 09:06:21 -0700254 for (size_t i = 0; i < frame.num_references; ++i) {
philipel3b3c9c42017-09-11 09:38:36 -0700255 if (frame.references[i] < 0 || frame.references[i] >= frame.picture_id)
philipel112adf92017-06-15 09:06:21 -0700256 return false;
philipel3b3c9c42017-09-11 09:38:36 -0700257
philipel112adf92017-06-15 09:06:21 -0700258 for (size_t j = i + 1; j < frame.num_references; ++j) {
259 if (frame.references[i] == frame.references[j])
260 return false;
261 }
262 }
263
264 if (frame.inter_layer_predicted && frame.spatial_layer == 0)
265 return false;
266
267 return true;
268}
269
gnishb2a318b2017-05-10 09:21:33 -0700270void FrameBuffer::UpdatePlayoutDelays(const FrameObject& frame) {
271 TRACE_EVENT0("webrtc", "FrameBuffer::UpdatePlayoutDelays");
272 PlayoutDelay playout_delay = frame.EncodedImage().playout_delay_;
273 if (playout_delay.min_ms >= 0)
274 timing_->set_min_playout_delay(playout_delay.min_ms);
275
276 if (playout_delay.max_ms >= 0)
277 timing_->set_max_playout_delay(playout_delay.max_ms);
278}
279
philipele0b2f152016-09-28 10:23:49 +0200280int FrameBuffer::InsertFrame(std::unique_ptr<FrameObject> frame) {
tommidb23ea62017-03-03 07:21:18 -0800281 TRACE_EVENT0("webrtc", "FrameBuffer::InsertFrame");
philipel93e451b2016-10-06 12:25:13 +0200282 RTC_DCHECK(frame);
philipela45102f2017-02-22 05:30:39 -0800283 if (stats_callback_)
ilnik6d5b4d62017-08-30 03:32:14 -0700284 stats_callback_->OnCompleteFrame(frame->is_keyframe(), frame->size(),
285 frame->contentType());
philipelbe7a9e52016-05-19 12:19:35 +0200286 FrameKey key(frame->picture_id, frame->spatial_layer);
tommi0a735642017-03-14 06:23:57 -0700287
288 rtc::CritScope lock(&crit_);
philipel29f730e2017-03-15 08:10:08 -0700289
philipele0b2f152016-09-28 10:23:49 +0200290 int last_continuous_picture_id =
291 last_continuous_frame_it_ == frames_.end()
292 ? -1
293 : last_continuous_frame_it_->first.picture_id;
294
philipel112adf92017-06-15 09:06:21 -0700295 if (!ValidReferences(*frame)) {
296 LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) (" << key.picture_id
297 << ":" << static_cast<int>(key.spatial_layer)
298 << ") has invalid frame references, dropping frame.";
299 return last_continuous_picture_id;
300 }
301
philipele0b2f152016-09-28 10:23:49 +0200302 if (num_frames_buffered_ >= kMaxFramesBuffered) {
303 LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) (" << key.picture_id
304 << ":" << static_cast<int>(key.spatial_layer)
305 << ") could not be inserted due to the frame "
306 << "buffer being full, dropping frame.";
307 return last_continuous_picture_id;
308 }
309
philipele0b2f152016-09-28 10:23:49 +0200310 if (last_decoded_frame_it_ != frames_.end() &&
philipelf6842692017-04-28 03:29:15 -0700311 key <= last_decoded_frame_it_->first) {
philipelfcc60062017-01-18 05:35:20 -0800312 if (AheadOf(frame->timestamp, last_decoded_frame_timestamp_) &&
philipel3042c2d2017-08-18 04:55:02 -0700313 frame->is_keyframe()) {
philipelfcc60062017-01-18 05:35:20 -0800314 // If this frame has a newer timestamp but an earlier picture id then we
315 // assume there has been a jump in the picture id due to some encoder
316 // reconfiguration or some other reason. Even though this is not according
317 // to spec we can still continue to decode from this frame if it is a
318 // keyframe.
319 LOG(LS_WARNING) << "A jump in picture id was detected, clearing buffer.";
320 ClearFramesAndHistory();
321 last_continuous_picture_id = -1;
322 } else {
323 LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) ("
324 << key.picture_id << ":"
325 << static_cast<int>(key.spatial_layer)
326 << ") inserted after frame ("
327 << last_decoded_frame_it_->first.picture_id << ":"
328 << static_cast<int>(
329 last_decoded_frame_it_->first.spatial_layer)
330 << ") was handed off for decoding, dropping frame.";
331 return last_continuous_picture_id;
332 }
philipele0b2f152016-09-28 10:23:49 +0200333 }
334
philipel146a48b2017-04-20 04:04:38 -0700335 // Test if inserting this frame would cause the order of the frames to become
336 // ambiguous (covering more than half the interval of 2^16). This can happen
337 // when the picture id make large jumps mid stream.
338 if (!frames_.empty() &&
339 key < frames_.begin()->first &&
340 frames_.rbegin()->first < key) {
341 LOG(LS_WARNING) << "A jump in picture id was detected, clearing buffer.";
342 ClearFramesAndHistory();
343 last_continuous_picture_id = -1;
344 }
345
philipele0b2f152016-09-28 10:23:49 +0200346 auto info = frames_.insert(std::make_pair(key, FrameInfo())).first;
347
philipel93e451b2016-10-06 12:25:13 +0200348 if (info->second.frame) {
349 LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) (" << key.picture_id
350 << ":" << static_cast<int>(key.spatial_layer)
351 << ") already inserted, dropping frame.";
philipele0b2f152016-09-28 10:23:49 +0200352 return last_continuous_picture_id;
353 }
354
philipel93e451b2016-10-06 12:25:13 +0200355 if (!UpdateFrameInfoWithIncomingFrame(*frame, info))
356 return last_continuous_picture_id;
gnishb2a318b2017-05-10 09:21:33 -0700357 UpdatePlayoutDelays(*frame);
philipele0b2f152016-09-28 10:23:49 +0200358 info->second.frame = std::move(frame);
359 ++num_frames_buffered_;
360
361 if (info->second.num_missing_continuous == 0) {
362 info->second.continuous = true;
363 PropagateContinuity(info);
364 last_continuous_picture_id = last_continuous_frame_it_->first.picture_id;
365
366 // Since we now have new continuous frames there might be a better frame
367 // to return from NextFrame. Signal that thread so that it again can choose
368 // which frame to return.
tommi0a735642017-03-14 06:23:57 -0700369 new_continuous_frame_event_.Set();
philipele0b2f152016-09-28 10:23:49 +0200370 }
371
372 return last_continuous_picture_id;
philipelbe7a9e52016-05-19 12:19:35 +0200373}
374
philipele0b2f152016-09-28 10:23:49 +0200375void FrameBuffer::PropagateContinuity(FrameMap::iterator start) {
tommidb23ea62017-03-03 07:21:18 -0800376 TRACE_EVENT0("webrtc", "FrameBuffer::PropagateContinuity");
philipele0b2f152016-09-28 10:23:49 +0200377 RTC_DCHECK(start->second.continuous);
378 if (last_continuous_frame_it_ == frames_.end())
379 last_continuous_frame_it_ = start;
380
381 std::queue<FrameMap::iterator> continuous_frames;
382 continuous_frames.push(start);
383
384 // A simple BFS to traverse continuous frames.
385 while (!continuous_frames.empty()) {
386 auto frame = continuous_frames.front();
387 continuous_frames.pop();
388
389 if (last_continuous_frame_it_->first < frame->first)
390 last_continuous_frame_it_ = frame;
391
392 // Loop through all dependent frames, and if that frame no longer has
393 // any unfulfilled dependencies then that frame is continuous as well.
394 for (size_t d = 0; d < frame->second.num_dependent_frames; ++d) {
395 auto frame_ref = frames_.find(frame->second.dependent_frames[d]);
philipel112adf92017-06-15 09:06:21 -0700396 RTC_DCHECK(frame_ref != frames_.end());
philipele0b2f152016-09-28 10:23:49 +0200397
philipel112adf92017-06-15 09:06:21 -0700398 // TODO(philipel): Look into why we've seen this happen.
399 if (frame_ref != frames_.end()) {
400 --frame_ref->second.num_missing_continuous;
401 if (frame_ref->second.num_missing_continuous == 0) {
402 frame_ref->second.continuous = true;
403 continuous_frames.push(frame_ref);
404 }
philipele0b2f152016-09-28 10:23:49 +0200405 }
406 }
407 }
408}
409
410void FrameBuffer::PropagateDecodability(const FrameInfo& info) {
tommidb23ea62017-03-03 07:21:18 -0800411 TRACE_EVENT0("webrtc", "FrameBuffer::PropagateDecodability");
tommie95b78b2017-05-14 07:23:11 -0700412 RTC_CHECK(info.num_dependent_frames < FrameInfo::kMaxNumDependentFrames);
philipele0b2f152016-09-28 10:23:49 +0200413 for (size_t d = 0; d < info.num_dependent_frames; ++d) {
414 auto ref_info = frames_.find(info.dependent_frames[d]);
philipel93e451b2016-10-06 12:25:13 +0200415 RTC_DCHECK(ref_info != frames_.end());
tommie95b78b2017-05-14 07:23:11 -0700416 // TODO(philipel): Look into why we've seen this happen.
417 if (ref_info != frames_.end()) {
418 RTC_DCHECK_GT(ref_info->second.num_missing_decodable, 0U);
419 --ref_info->second.num_missing_decodable;
420 }
philipele0b2f152016-09-28 10:23:49 +0200421 }
422}
423
424void FrameBuffer::AdvanceLastDecodedFrame(FrameMap::iterator decoded) {
tommidb23ea62017-03-03 07:21:18 -0800425 TRACE_EVENT0("webrtc", "FrameBuffer::AdvanceLastDecodedFrame");
philipele0b2f152016-09-28 10:23:49 +0200426 if (last_decoded_frame_it_ == frames_.end()) {
427 last_decoded_frame_it_ = frames_.begin();
428 } else {
429 RTC_DCHECK(last_decoded_frame_it_->first < decoded->first);
430 ++last_decoded_frame_it_;
431 }
432 --num_frames_buffered_;
433 ++num_frames_history_;
434
435 // First, delete non-decoded frames from the history.
436 while (last_decoded_frame_it_ != decoded) {
437 if (last_decoded_frame_it_->second.frame)
438 --num_frames_buffered_;
439 last_decoded_frame_it_ = frames_.erase(last_decoded_frame_it_);
philipelbe7a9e52016-05-19 12:19:35 +0200440 }
441
philipele0b2f152016-09-28 10:23:49 +0200442 // Then remove old history if we have too much history saved.
443 if (num_frames_history_ > kMaxFramesHistory) {
444 frames_.erase(frames_.begin());
445 --num_frames_history_;
446 }
447}
448
449bool FrameBuffer::UpdateFrameInfoWithIncomingFrame(const FrameObject& frame,
450 FrameMap::iterator info) {
tommidb23ea62017-03-03 07:21:18 -0800451 TRACE_EVENT0("webrtc", "FrameBuffer::UpdateFrameInfoWithIncomingFrame");
philipele0b2f152016-09-28 10:23:49 +0200452 FrameKey key(frame.picture_id, frame.spatial_layer);
453 info->second.num_missing_continuous = frame.num_references;
454 info->second.num_missing_decodable = frame.num_references;
455
456 RTC_DCHECK(last_decoded_frame_it_ == frames_.end() ||
457 last_decoded_frame_it_->first < info->first);
458
459 // Check how many dependencies that have already been fulfilled.
460 for (size_t i = 0; i < frame.num_references; ++i) {
461 FrameKey ref_key(frame.references[i], frame.spatial_layer);
462 auto ref_info = frames_.find(ref_key);
463
464 // Does |frame| depend on a frame earlier than the last decoded frame?
465 if (last_decoded_frame_it_ != frames_.end() &&
466 ref_key <= last_decoded_frame_it_->first) {
467 if (ref_info == frames_.end()) {
philipel65e1f942017-07-24 08:26:53 -0700468 int64_t now_ms = clock_->TimeInMilliseconds();
469 if (last_log_non_decoded_ms_ + kLogNonDecodedIntervalMs < now_ms) {
470 LOG(LS_WARNING)
471 << "Frame with (picture_id:spatial_id) (" << key.picture_id << ":"
472 << static_cast<int>(key.spatial_layer)
473 << ") depends on a non-decoded frame more previous than"
474 << " the last decoded frame, dropping frame.";
475 last_log_non_decoded_ms_ = now_ms;
476 }
philipele0b2f152016-09-28 10:23:49 +0200477 return false;
478 }
479
480 --info->second.num_missing_continuous;
481 --info->second.num_missing_decodable;
482 } else {
483 if (ref_info == frames_.end())
484 ref_info = frames_.insert(std::make_pair(ref_key, FrameInfo())).first;
485
486 if (ref_info->second.continuous)
487 --info->second.num_missing_continuous;
488
489 // Add backwards reference so |frame| can be updated when new
490 // frames are inserted or decoded.
491 ref_info->second.dependent_frames[ref_info->second.num_dependent_frames] =
492 key;
tommie95b78b2017-05-14 07:23:11 -0700493 RTC_DCHECK_LT(ref_info->second.num_dependent_frames,
494 (FrameInfo::kMaxNumDependentFrames - 1));
495 // TODO(philipel): Look into why this could happen and handle
496 // appropriately.
497 if (ref_info->second.num_dependent_frames <
498 (FrameInfo::kMaxNumDependentFrames - 1)) {
499 ++ref_info->second.num_dependent_frames;
500 }
philipele0b2f152016-09-28 10:23:49 +0200501 }
philipel93e451b2016-10-06 12:25:13 +0200502 RTC_DCHECK_LE(ref_info->second.num_missing_continuous,
503 ref_info->second.num_missing_decodable);
philipelbe7a9e52016-05-19 12:19:35 +0200504 }
505
philipele0b2f152016-09-28 10:23:49 +0200506 // Check if we have the lower spatial layer frame.
philipelbe7a9e52016-05-19 12:19:35 +0200507 if (frame.inter_layer_predicted) {
philipele0b2f152016-09-28 10:23:49 +0200508 ++info->second.num_missing_continuous;
509 ++info->second.num_missing_decodable;
510
philipelbe7a9e52016-05-19 12:19:35 +0200511 FrameKey ref_key(frame.picture_id, frame.spatial_layer - 1);
philipele0b2f152016-09-28 10:23:49 +0200512 // Gets or create the FrameInfo for the referenced frame.
513 auto ref_info = frames_.insert(std::make_pair(ref_key, FrameInfo())).first;
514 if (ref_info->second.continuous)
515 --info->second.num_missing_continuous;
516
517 if (ref_info == last_decoded_frame_it_) {
518 --info->second.num_missing_decodable;
519 } else {
520 ref_info->second.dependent_frames[ref_info->second.num_dependent_frames] =
521 key;
522 ++ref_info->second.num_dependent_frames;
523 }
philipel93e451b2016-10-06 12:25:13 +0200524 RTC_DCHECK_LE(ref_info->second.num_missing_continuous,
525 ref_info->second.num_missing_decodable);
philipelbe7a9e52016-05-19 12:19:35 +0200526 }
527
philipel93e451b2016-10-06 12:25:13 +0200528 RTC_DCHECK_LE(info->second.num_missing_continuous,
529 info->second.num_missing_decodable);
530
philipelbe7a9e52016-05-19 12:19:35 +0200531 return true;
532}
533
philipelbe742702016-11-30 01:31:40 -0800534void FrameBuffer::UpdateJitterDelay() {
tommidb23ea62017-03-03 07:21:18 -0800535 TRACE_EVENT0("webrtc", "FrameBuffer::UpdateJitterDelay");
philipela45102f2017-02-22 05:30:39 -0800536 if (!stats_callback_)
537 return;
philipelbe742702016-11-30 01:31:40 -0800538
philipela45102f2017-02-22 05:30:39 -0800539 int decode_ms;
540 int max_decode_ms;
541 int current_delay_ms;
542 int target_delay_ms;
543 int jitter_buffer_ms;
544 int min_playout_delay_ms;
545 int render_delay_ms;
546 if (timing_->GetTimings(&decode_ms, &max_decode_ms, &current_delay_ms,
547 &target_delay_ms, &jitter_buffer_ms,
548 &min_playout_delay_ms, &render_delay_ms)) {
549 stats_callback_->OnFrameBufferTimingsUpdated(
550 decode_ms, max_decode_ms, current_delay_ms, target_delay_ms,
551 jitter_buffer_ms, min_playout_delay_ms, render_delay_ms);
philipelbe742702016-11-30 01:31:40 -0800552 }
philipel266f0a42016-11-28 08:49:07 -0800553}
554
ilnik2edc6842017-07-06 03:06:50 -0700555void FrameBuffer::UpdateTimingFrameInfo() {
556 TRACE_EVENT0("webrtc", "FrameBuffer::UpdateTimingFrameInfo");
557 rtc::Optional<TimingFrameInfo> info = timing_->GetTimingFrameInfo();
558 if (info)
559 stats_callback_->OnTimingFrameInfoUpdated(*info);
560}
561
philipelfcc60062017-01-18 05:35:20 -0800562void FrameBuffer::ClearFramesAndHistory() {
ilnik2edc6842017-07-06 03:06:50 -0700563 TRACE_EVENT0("webrtc", "FrameBuffer::ClearFramesAndHistory");
philipelfcc60062017-01-18 05:35:20 -0800564 frames_.clear();
565 last_decoded_frame_it_ = frames_.end();
566 last_continuous_frame_it_ = frames_.end();
philipel1c056252017-01-31 09:53:12 -0800567 next_frame_it_ = frames_.end();
philipelfcc60062017-01-18 05:35:20 -0800568 num_frames_history_ = 0;
569 num_frames_buffered_ = 0;
570}
571
philipelbe7a9e52016-05-19 12:19:35 +0200572} // namespace video_coding
573} // namespace webrtc