blob: 832fd360f041f85df8d1292309a829b1cb6c9873 [file] [log] [blame]
philipelbe7a9e52016-05-19 12:19:35 +02001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "modules/video_coding/frame_buffer2.h"
philipelbe7a9e52016-05-19 12:19:35 +020012
13#include <algorithm>
Yves Gerey3e707812018-11-28 16:47:49 +010014#include <cstdlib>
15#include <iterator>
philipele0b2f152016-09-28 10:23:49 +020016#include <queue>
Yves Gerey3e707812018-11-28 16:47:49 +010017#include <utility>
philipel798b2822018-06-11 13:10:14 +020018#include <vector>
philipelbe7a9e52016-05-19 12:19:35 +020019
Yves Gerey3e707812018-11-28 16:47:49 +010020#include "api/video/encoded_image.h"
21#include "api/video/video_timing.h"
22#include "common_types.h" // NOLINT(build/include)
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "modules/video_coding/include/video_coding_defines.h"
24#include "modules/video_coding/jitter_estimator.h"
25#include "modules/video_coding/timing.h"
26#include "rtc_base/checks.h"
Yves Gerey3e707812018-11-28 16:47:49 +010027#include "rtc_base/experiments/rtt_mult_experiment.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020028#include "rtc_base/logging.h"
Yves Gerey3e707812018-11-28 16:47:49 +010029#include "rtc_base/numerics/sequence_number_util.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020030#include "rtc_base/trace_event.h"
31#include "system_wrappers/include/clock.h"
philipel707f2782017-10-02 14:10:28 +020032#include "system_wrappers/include/field_trial.h"
philipelbe7a9e52016-05-19 12:19:35 +020033
34namespace webrtc {
35namespace video_coding {
36
37namespace {
philipele0b2f152016-09-28 10:23:49 +020038// Max number of frames the buffer will hold.
Ilya Nikolaevskiy13717842019-01-14 13:24:22 +010039constexpr size_t kMaxFramesBuffered = 800;
philipelbe7a9e52016-05-19 12:19:35 +020040
philipele0b2f152016-09-28 10:23:49 +020041// Max number of decoded frame info that will be saved.
Ilya Nikolaevskiy13717842019-01-14 13:24:22 +010042constexpr int kMaxFramesHistory = 1 << 13;
philipel65e1f942017-07-24 08:26:53 -070043
Ilya Nikolaevskiy8c4fe162018-02-27 15:49:47 +010044// The time it's allowed for a frame to be late to its rendering prediction and
45// still be rendered.
Ilya Nikolaevskiy7eef0072018-02-28 09:59:26 +010046constexpr int kMaxAllowedFrameDelayMs = 5;
Ilya Nikolaevskiy8c4fe162018-02-27 15:49:47 +010047
philipel65e1f942017-07-24 08:26:53 -070048constexpr int64_t kLogNonDecodedIntervalMs = 5000;
philipelbe7a9e52016-05-19 12:19:35 +020049} // namespace
50
philipelbe7a9e52016-05-19 12:19:35 +020051FrameBuffer::FrameBuffer(Clock* clock,
52 VCMJitterEstimator* jitter_estimator,
philipela45102f2017-02-22 05:30:39 -080053 VCMTiming* timing,
54 VCMReceiveStatisticsCallback* stats_callback)
Ilya Nikolaevskiy13717842019-01-14 13:24:22 +010055 : decoded_frames_history_(kMaxFramesHistory),
56 clock_(clock),
philipelbe7a9e52016-05-19 12:19:35 +020057 jitter_estimator_(jitter_estimator),
58 timing_(timing),
philipel4f6cd6a2016-08-03 10:59:32 +020059 inter_frame_delay_(clock_->TimeInMilliseconds()),
philipel29f730e2017-03-15 08:10:08 -070060 stopped_(false),
philipela45102f2017-02-22 05:30:39 -080061 protection_mode_(kProtectionNack),
philipel65e1f942017-07-24 08:26:53 -070062 stats_callback_(stats_callback),
63 last_log_non_decoded_ms_(-kLogNonDecodedIntervalMs) {}
philipel266f0a42016-11-28 08:49:07 -080064
philipela45102f2017-02-22 05:30:39 -080065FrameBuffer::~FrameBuffer() {}
philipelbe7a9e52016-05-19 12:19:35 +020066
philipel75562822016-09-05 10:57:41 +020067FrameBuffer::ReturnReason FrameBuffer::NextFrame(
68 int64_t max_wait_time_ms,
philipele7c891f2018-02-22 14:35:06 +010069 std::unique_ptr<EncodedFrame>* frame_out,
philipel3042c2d2017-08-18 04:55:02 -070070 bool keyframe_required) {
tommidb23ea62017-03-03 07:21:18 -080071 TRACE_EVENT0("webrtc", "FrameBuffer::NextFrame");
philipel1c056252017-01-31 09:53:12 -080072 int64_t latest_return_time_ms =
73 clock_->TimeInMilliseconds() + max_wait_time_ms;
philipel504c47d2016-06-30 17:33:02 +020074 int64_t wait_ms = max_wait_time_ms;
philipel29f730e2017-03-15 08:10:08 -070075 int64_t now_ms = 0;
philipele0b2f152016-09-28 10:23:49 +020076
77 do {
philipel29f730e2017-03-15 08:10:08 -070078 now_ms = clock_->TimeInMilliseconds();
philipel504c47d2016-06-30 17:33:02 +020079 {
80 rtc::CritScope lock(&crit_);
tommi0a735642017-03-14 06:23:57 -070081 new_continuous_frame_event_.Reset();
philipel29f730e2017-03-15 08:10:08 -070082 if (stopped_)
83 return kStopped;
84
85 wait_ms = max_wait_time_ms;
86
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +010087 // Need to hold |crit_| in order to access frames_to_decode_. therefore we
philipele0b2f152016-09-28 10:23:49 +020088 // set it here in the loop instead of outside the loop in order to not
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +010089 // acquire the lock unnecessarily.
90 frames_to_decode_.clear();
philipelbe7a9e52016-05-19 12:19:35 +020091
Ilya Nikolaevskiy6551faf2019-01-10 15:16:47 +010092 // |last_continuous_frame_| may be empty below, but nullopt is smaller
93 // than everything else and loop will immediately terminate as expected.
94 for (auto frame_it = frames_.begin();
95 frame_it != frames_.end() &&
96 frame_it->first <= last_continuous_frame_;
philipel146a48b2017-04-20 04:04:38 -070097 ++frame_it) {
philipel93e451b2016-10-06 12:25:13 +020098 if (!frame_it->second.continuous ||
99 frame_it->second.num_missing_decodable > 0) {
philipele0b2f152016-09-28 10:23:49 +0200100 continue;
philipel93e451b2016-10-06 12:25:13 +0200101 }
philipele0b2f152016-09-28 10:23:49 +0200102
philipele7c891f2018-02-22 14:35:06 +0100103 EncodedFrame* frame = frame_it->second.frame.get();
philipel3042c2d2017-08-18 04:55:02 -0700104
105 if (keyframe_required && !frame->is_keyframe())
106 continue;
107
Ilya Nikolaevskiy13717842019-01-14 13:24:22 +0100108 auto last_decoded_frame_timestamp =
109 decoded_frames_history_.GetLastDecodedFrameTimestamp();
110
Ilya Nikolaevskiye6a2d942018-11-07 14:32:28 +0100111 // TODO(https://bugs.webrtc.org/9974): consider removing this check
112 // as it may make a stream undecodable after a very long delay between
113 // frames.
Ilya Nikolaevskiy13717842019-01-14 13:24:22 +0100114 if (last_decoded_frame_timestamp &&
115 AheadOf(*last_decoded_frame_timestamp, frame->Timestamp())) {
philipel6d216502018-10-22 14:36:45 +0200116 continue;
117 }
118
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100119 // Only ever return all parts of a superframe. Therefore skip this
120 // frame if it's not a beginning of a superframe.
121 if (frame->inter_layer_predicted) {
122 continue;
123 }
124
125 // Gather all remaining frames for the same superframe.
126 std::vector<FrameMap::iterator> current_superframe;
127 current_superframe.push_back(frame_it);
128 bool last_layer_completed =
129 frame_it->second.frame->is_last_spatial_layer;
130 FrameMap::iterator next_frame_it = frame_it;
131 while (true) {
132 ++next_frame_it;
133 if (next_frame_it == frames_.end() ||
134 next_frame_it->first.picture_id != frame->id.picture_id ||
135 !next_frame_it->second.continuous) {
136 break;
137 }
138 // Check if the next frame has some undecoded references other than
139 // the previous frame in the same superframe.
140 size_t num_allowed_undecoded_refs =
141 (next_frame_it->second.frame->inter_layer_predicted) ? 1 : 0;
142 if (next_frame_it->second.num_missing_decodable >
143 num_allowed_undecoded_refs) {
144 break;
145 }
146 // All frames in the superframe should have the same timestamp.
147 if (frame->Timestamp() != next_frame_it->second.frame->Timestamp()) {
148 RTC_LOG(LS_WARNING)
149 << "Frames in a single superframe have different"
150 " timestamps. Skipping undecodable superframe.";
151 break;
152 }
153 current_superframe.push_back(next_frame_it);
154 last_layer_completed =
155 next_frame_it->second.frame->is_last_spatial_layer;
156 }
157 // Check if the current superframe is complete.
158 // TODO(bugs.webrtc.org/10064): consider returning all available to
159 // decode frames even if the superframe is not complete yet.
160 if (!last_layer_completed) {
161 continue;
162 }
163
164 frames_to_decode_ = std::move(current_superframe);
165
philipel6d216502018-10-22 14:36:45 +0200166 if (frame->RenderTime() == -1) {
Niels Möller23775882018-08-16 10:24:12 +0200167 frame->SetRenderTime(
168 timing_->RenderTimeMs(frame->Timestamp(), now_ms));
philipel6d216502018-10-22 14:36:45 +0200169 }
philipele0b2f152016-09-28 10:23:49 +0200170 wait_ms = timing_->MaxWaitingTime(frame->RenderTime(), now_ms);
171
172 // This will cause the frame buffer to prefer high framerate rather
173 // than high resolution in the case of the decoder not decoding fast
174 // enough and the stream has multiple spatial and temporal layers.
Ilya Nikolaevskiy8c4fe162018-02-27 15:49:47 +0100175 // For multiple temporal layers it may cause non-base layer frames to be
176 // skipped if they are late.
Ilya Nikolaevskiy7eef0072018-02-28 09:59:26 +0100177 if (wait_ms < -kMaxAllowedFrameDelayMs)
philipele0b2f152016-09-28 10:23:49 +0200178 continue;
179
180 break;
181 }
182 } // rtc::Critscope lock(&crit_);
183
philipel1c056252017-01-31 09:53:12 -0800184 wait_ms = std::min<int64_t>(wait_ms, latest_return_time_ms - now_ms);
philipele0b2f152016-09-28 10:23:49 +0200185 wait_ms = std::max<int64_t>(wait_ms, 0);
tommi0a735642017-03-14 06:23:57 -0700186 } while (new_continuous_frame_event_.Wait(wait_ms));
philipele0b2f152016-09-28 10:23:49 +0200187
philipel29f730e2017-03-15 08:10:08 -0700188 {
189 rtc::CritScope lock(&crit_);
190 now_ms = clock_->TimeInMilliseconds();
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100191 std::vector<EncodedFrame*> frames_out;
Ilya Nikolaevskiy13717842019-01-14 13:24:22 +0100192 for (FrameMap::iterator& frame_it : frames_to_decode_) {
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100193 RTC_DCHECK(frame_it != frames_.end());
194 EncodedFrame* frame = frame_it->second.frame.release();
philipele0b2f152016-09-28 10:23:49 +0200195
philipel29f730e2017-03-15 08:10:08 -0700196 if (!frame->delayed_by_retransmission()) {
197 int64_t frame_delay;
philipele0754302017-01-25 08:56:23 -0800198
Niels Möller23775882018-08-16 10:24:12 +0200199 if (inter_frame_delay_.CalculateDelay(frame->Timestamp(), &frame_delay,
philipel29f730e2017-03-15 08:10:08 -0700200 frame->ReceivedTime())) {
201 jitter_estimator_->UpdateEstimate(frame_delay, frame->size());
202 }
203
204 float rtt_mult = protection_mode_ == kProtectionNackFEC ? 0.0 : 1.0;
“Michaelf9fc1712018-08-27 10:08:58 -0500205 if (RttMultExperiment::RttMultEnabled()) {
206 rtt_mult = RttMultExperiment::GetRttMultValue();
207 }
philipel29f730e2017-03-15 08:10:08 -0700208 timing_->SetJitterDelay(jitter_estimator_->GetJitterEstimate(rtt_mult));
209 timing_->UpdateCurrentDelay(frame->RenderTime(), now_ms);
philipele21be1d2017-09-25 06:37:12 -0700210 } else {
“Michaelf9fc1712018-08-27 10:08:58 -0500211 if (RttMultExperiment::RttMultEnabled() ||
212 webrtc::field_trial::IsEnabled("WebRTC-AddRttToPlayoutDelay"))
philipel707f2782017-10-02 14:10:28 +0200213 jitter_estimator_->FrameNacked();
philipele0754302017-01-25 08:56:23 -0800214 }
215
stefan95e97542017-05-23 09:52:18 -0700216 // Gracefully handle bad RTP timestamps and render time issues.
217 if (HasBadRenderTiming(*frame, now_ms)) {
218 jitter_estimator_->Reset();
219 timing_->Reset();
Niels Möller23775882018-08-16 10:24:12 +0200220 frame->SetRenderTime(timing_->RenderTimeMs(frame->Timestamp(), now_ms));
stefan95e97542017-05-23 09:52:18 -0700221 }
222
philipel29f730e2017-03-15 08:10:08 -0700223 UpdateJitterDelay();
ilnik2edc6842017-07-06 03:06:50 -0700224 UpdateTimingFrameInfo();
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100225 PropagateDecodability(frame_it->second);
brandtr9078d8c2017-04-27 07:07:27 -0700226
Ilya Nikolaevskiy13717842019-01-14 13:24:22 +0100227 decoded_frames_history_.InsertDecoded(frame_it->first,
228 frame->Timestamp());
229
230 // Remove decoded frame and all undecoded frames before it.
231 frames_.erase(frames_.begin(), ++frame_it);
232
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100233 frames_out.push_back(frame);
234 }
235
236 if (!frames_out.empty()) {
237 if (frames_out.size() == 1) {
238 frame_out->reset(frames_out[0]);
239 } else {
240 frame_out->reset(CombineAndDeleteFrames(frames_out));
241 }
philipel29f730e2017-03-15 08:10:08 -0700242 return kFrameFound;
philipelbe7a9e52016-05-19 12:19:35 +0200243 }
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100244 } // rtc::Critscope lock(&crit_)
tommi0a735642017-03-14 06:23:57 -0700245
246 if (latest_return_time_ms - now_ms > 0) {
philipel1c056252017-01-31 09:53:12 -0800247 // If |next_frame_it_ == frames_.end()| and there is still time left, it
248 // means that the frame buffer was cleared as the thread in this function
249 // was waiting to acquire |crit_| in order to return. Wait for the
250 // remaining time and then return.
251 return NextFrame(latest_return_time_ms - now_ms, frame_out);
philipelbe7a9e52016-05-19 12:19:35 +0200252 }
tommi0a735642017-03-14 06:23:57 -0700253 return kTimeout;
philipelbe7a9e52016-05-19 12:19:35 +0200254}
255
philipele7c891f2018-02-22 14:35:06 +0100256bool FrameBuffer::HasBadRenderTiming(const EncodedFrame& frame,
257 int64_t now_ms) {
stefan95e97542017-05-23 09:52:18 -0700258 // Assume that render timing errors are due to changes in the video stream.
259 int64_t render_time_ms = frame.RenderTimeMs();
Stefan Holmer812ceaf2018-05-15 13:00:10 +0200260 // Zero render time means render immediately.
261 if (render_time_ms == 0) {
262 return false;
263 }
stefan95e97542017-05-23 09:52:18 -0700264 if (render_time_ms < 0) {
265 return true;
266 }
Stefan Holmer812ceaf2018-05-15 13:00:10 +0200267 const int64_t kMaxVideoDelayMs = 10000;
stefan95e97542017-05-23 09:52:18 -0700268 if (std::abs(render_time_ms - now_ms) > kMaxVideoDelayMs) {
269 int frame_delay = static_cast<int>(std::abs(render_time_ms - now_ms));
Mirko Bonadei675513b2017-11-09 11:09:25 +0100270 RTC_LOG(LS_WARNING)
271 << "A frame about to be decoded is out of the configured "
272 << "delay bounds (" << frame_delay << " > " << kMaxVideoDelayMs
273 << "). Resetting the video jitter buffer.";
stefan95e97542017-05-23 09:52:18 -0700274 return true;
275 }
276 if (static_cast<int>(timing_->TargetVideoDelay()) > kMaxVideoDelayMs) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100277 RTC_LOG(LS_WARNING) << "The video target delay has grown larger than "
278 << kMaxVideoDelayMs << " ms.";
stefan95e97542017-05-23 09:52:18 -0700279 return true;
280 }
281 return false;
282}
283
philipel4f6cd6a2016-08-03 10:59:32 +0200284void FrameBuffer::SetProtectionMode(VCMVideoProtection mode) {
tommidb23ea62017-03-03 07:21:18 -0800285 TRACE_EVENT0("webrtc", "FrameBuffer::SetProtectionMode");
philipel4f6cd6a2016-08-03 10:59:32 +0200286 rtc::CritScope lock(&crit_);
287 protection_mode_ = mode;
288}
289
philipel504c47d2016-06-30 17:33:02 +0200290void FrameBuffer::Start() {
tommidb23ea62017-03-03 07:21:18 -0800291 TRACE_EVENT0("webrtc", "FrameBuffer::Start");
philipel29f730e2017-03-15 08:10:08 -0700292 rtc::CritScope lock(&crit_);
293 stopped_ = false;
philipel504c47d2016-06-30 17:33:02 +0200294}
295
296void FrameBuffer::Stop() {
tommidb23ea62017-03-03 07:21:18 -0800297 TRACE_EVENT0("webrtc", "FrameBuffer::Stop");
philipel29f730e2017-03-15 08:10:08 -0700298 rtc::CritScope lock(&crit_);
299 stopped_ = true;
tommi0a735642017-03-14 06:23:57 -0700300 new_continuous_frame_event_.Set();
philipel504c47d2016-06-30 17:33:02 +0200301}
302
Ilya Nikolaevskiye6a2d942018-11-07 14:32:28 +0100303void FrameBuffer::Clear() {
304 rtc::CritScope lock(&crit_);
305 ClearFramesAndHistory();
306}
307
philipele21be1d2017-09-25 06:37:12 -0700308void FrameBuffer::UpdateRtt(int64_t rtt_ms) {
309 rtc::CritScope lock(&crit_);
310 jitter_estimator_->UpdateRtt(rtt_ms);
311}
312
philipele7c891f2018-02-22 14:35:06 +0100313bool FrameBuffer::ValidReferences(const EncodedFrame& frame) const {
philipel0fa82a62018-03-19 15:34:53 +0100314 if (frame.id.picture_id < 0)
philipel3b3c9c42017-09-11 09:38:36 -0700315 return false;
316
philipel112adf92017-06-15 09:06:21 -0700317 for (size_t i = 0; i < frame.num_references; ++i) {
philipel0fa82a62018-03-19 15:34:53 +0100318 if (frame.references[i] < 0 || frame.references[i] >= frame.id.picture_id)
philipel112adf92017-06-15 09:06:21 -0700319 return false;
philipel3b3c9c42017-09-11 09:38:36 -0700320
philipel112adf92017-06-15 09:06:21 -0700321 for (size_t j = i + 1; j < frame.num_references; ++j) {
322 if (frame.references[i] == frame.references[j])
323 return false;
324 }
325 }
326
philipel0fa82a62018-03-19 15:34:53 +0100327 if (frame.inter_layer_predicted && frame.id.spatial_layer == 0)
philipel112adf92017-06-15 09:06:21 -0700328 return false;
329
330 return true;
331}
332
philipele7c891f2018-02-22 14:35:06 +0100333void FrameBuffer::UpdatePlayoutDelays(const EncodedFrame& frame) {
gnishb2a318b2017-05-10 09:21:33 -0700334 TRACE_EVENT0("webrtc", "FrameBuffer::UpdatePlayoutDelays");
335 PlayoutDelay playout_delay = frame.EncodedImage().playout_delay_;
336 if (playout_delay.min_ms >= 0)
337 timing_->set_min_playout_delay(playout_delay.min_ms);
338
339 if (playout_delay.max_ms >= 0)
340 timing_->set_max_playout_delay(playout_delay.max_ms);
philipel0a9f6de2018-02-28 11:29:47 +0100341
342 if (!frame.delayed_by_retransmission())
Niels Möller23775882018-08-16 10:24:12 +0200343 timing_->IncomingTimestamp(frame.Timestamp(), frame.ReceivedTime());
gnishb2a318b2017-05-10 09:21:33 -0700344}
345
philipele7c891f2018-02-22 14:35:06 +0100346int64_t FrameBuffer::InsertFrame(std::unique_ptr<EncodedFrame> frame) {
tommidb23ea62017-03-03 07:21:18 -0800347 TRACE_EVENT0("webrtc", "FrameBuffer::InsertFrame");
philipel93e451b2016-10-06 12:25:13 +0200348 RTC_DCHECK(frame);
philipela45102f2017-02-22 05:30:39 -0800349 if (stats_callback_)
ilnik6d5b4d62017-08-30 03:32:14 -0700350 stats_callback_->OnCompleteFrame(frame->is_keyframe(), frame->size(),
351 frame->contentType());
philipel0fa82a62018-03-19 15:34:53 +0100352 const VideoLayerFrameId& id = frame->id;
tommi0a735642017-03-14 06:23:57 -0700353
354 rtc::CritScope lock(&crit_);
philipel29f730e2017-03-15 08:10:08 -0700355
philipel1610f942017-12-12 13:58:31 +0100356 int64_t last_continuous_picture_id =
Ilya Nikolaevskiy6551faf2019-01-10 15:16:47 +0100357 !last_continuous_frame_ ? -1 : last_continuous_frame_->picture_id;
philipele0b2f152016-09-28 10:23:49 +0200358
philipel112adf92017-06-15 09:06:21 -0700359 if (!ValidReferences(*frame)) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100360 RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) ("
philipel0fa82a62018-03-19 15:34:53 +0100361 << id.picture_id << ":"
362 << static_cast<int>(id.spatial_layer)
Mirko Bonadei675513b2017-11-09 11:09:25 +0100363 << ") has invalid frame references, dropping frame.";
philipel112adf92017-06-15 09:06:21 -0700364 return last_continuous_picture_id;
365 }
366
Ilya Nikolaevskiy6551faf2019-01-10 15:16:47 +0100367 if (frames_.size() >= kMaxFramesBuffered) {
philipel9771c502018-03-02 11:06:27 +0100368 if (frame->is_keyframe()) {
369 RTC_LOG(LS_WARNING) << "Inserting keyframe (picture_id:spatial_id) ("
philipel0fa82a62018-03-19 15:34:53 +0100370 << id.picture_id << ":"
371 << static_cast<int>(id.spatial_layer)
philipel9771c502018-03-02 11:06:27 +0100372 << ") but buffer is full, clearing"
373 << " buffer and inserting the frame.";
374 ClearFramesAndHistory();
375 } else {
376 RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) ("
philipel0fa82a62018-03-19 15:34:53 +0100377 << id.picture_id << ":"
378 << static_cast<int>(id.spatial_layer)
philipel9771c502018-03-02 11:06:27 +0100379 << ") could not be inserted due to the frame "
380 << "buffer being full, dropping frame.";
381 return last_continuous_picture_id;
382 }
philipele0b2f152016-09-28 10:23:49 +0200383 }
384
Ilya Nikolaevskiy13717842019-01-14 13:24:22 +0100385 auto last_decoded_frame = decoded_frames_history_.GetLastDecodedFrameId();
386 auto last_decoded_frame_timestamp =
387 decoded_frames_history_.GetLastDecodedFrameTimestamp();
388 if (last_decoded_frame && id <= *last_decoded_frame) {
389 if (AheadOf(frame->Timestamp(), *last_decoded_frame_timestamp) &&
philipel3042c2d2017-08-18 04:55:02 -0700390 frame->is_keyframe()) {
philipelfcc60062017-01-18 05:35:20 -0800391 // If this frame has a newer timestamp but an earlier picture id then we
392 // assume there has been a jump in the picture id due to some encoder
393 // reconfiguration or some other reason. Even though this is not according
394 // to spec we can still continue to decode from this frame if it is a
395 // keyframe.
Mirko Bonadei675513b2017-11-09 11:09:25 +0100396 RTC_LOG(LS_WARNING)
397 << "A jump in picture id was detected, clearing buffer.";
philipelfcc60062017-01-18 05:35:20 -0800398 ClearFramesAndHistory();
399 last_continuous_picture_id = -1;
400 } else {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100401 RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) ("
philipel0fa82a62018-03-19 15:34:53 +0100402 << id.picture_id << ":"
403 << static_cast<int>(id.spatial_layer)
Mirko Bonadei675513b2017-11-09 11:09:25 +0100404 << ") inserted after frame ("
Ilya Nikolaevskiy13717842019-01-14 13:24:22 +0100405 << last_decoded_frame->picture_id << ":"
406 << static_cast<int>(last_decoded_frame->spatial_layer)
Mirko Bonadei675513b2017-11-09 11:09:25 +0100407 << ") was handed off for decoding, dropping frame.";
philipelfcc60062017-01-18 05:35:20 -0800408 return last_continuous_picture_id;
409 }
philipele0b2f152016-09-28 10:23:49 +0200410 }
411
philipel146a48b2017-04-20 04:04:38 -0700412 // Test if inserting this frame would cause the order of the frames to become
413 // ambiguous (covering more than half the interval of 2^16). This can happen
414 // when the picture id make large jumps mid stream.
philipel0fa82a62018-03-19 15:34:53 +0100415 if (!frames_.empty() && id < frames_.begin()->first &&
416 frames_.rbegin()->first < id) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100417 RTC_LOG(LS_WARNING)
418 << "A jump in picture id was detected, clearing buffer.";
philipel146a48b2017-04-20 04:04:38 -0700419 ClearFramesAndHistory();
420 last_continuous_picture_id = -1;
421 }
422
philipel0fa82a62018-03-19 15:34:53 +0100423 auto info = frames_.emplace(id, FrameInfo()).first;
philipele0b2f152016-09-28 10:23:49 +0200424
philipel93e451b2016-10-06 12:25:13 +0200425 if (info->second.frame) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100426 RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) ("
philipel0fa82a62018-03-19 15:34:53 +0100427 << id.picture_id << ":"
428 << static_cast<int>(id.spatial_layer)
Mirko Bonadei675513b2017-11-09 11:09:25 +0100429 << ") already inserted, dropping frame.";
philipele0b2f152016-09-28 10:23:49 +0200430 return last_continuous_picture_id;
431 }
432
philipel93e451b2016-10-06 12:25:13 +0200433 if (!UpdateFrameInfoWithIncomingFrame(*frame, info))
434 return last_continuous_picture_id;
gnishb2a318b2017-05-10 09:21:33 -0700435 UpdatePlayoutDelays(*frame);
philipel0a9f6de2018-02-28 11:29:47 +0100436
philipele0b2f152016-09-28 10:23:49 +0200437 info->second.frame = std::move(frame);
philipele0b2f152016-09-28 10:23:49 +0200438
439 if (info->second.num_missing_continuous == 0) {
440 info->second.continuous = true;
441 PropagateContinuity(info);
Ilya Nikolaevskiy6551faf2019-01-10 15:16:47 +0100442 last_continuous_picture_id = last_continuous_frame_->picture_id;
philipele0b2f152016-09-28 10:23:49 +0200443
444 // Since we now have new continuous frames there might be a better frame
445 // to return from NextFrame. Signal that thread so that it again can choose
446 // which frame to return.
tommi0a735642017-03-14 06:23:57 -0700447 new_continuous_frame_event_.Set();
philipele0b2f152016-09-28 10:23:49 +0200448 }
449
450 return last_continuous_picture_id;
philipelbe7a9e52016-05-19 12:19:35 +0200451}
452
philipele0b2f152016-09-28 10:23:49 +0200453void FrameBuffer::PropagateContinuity(FrameMap::iterator start) {
tommidb23ea62017-03-03 07:21:18 -0800454 TRACE_EVENT0("webrtc", "FrameBuffer::PropagateContinuity");
philipele0b2f152016-09-28 10:23:49 +0200455 RTC_DCHECK(start->second.continuous);
philipele0b2f152016-09-28 10:23:49 +0200456
457 std::queue<FrameMap::iterator> continuous_frames;
458 continuous_frames.push(start);
459
460 // A simple BFS to traverse continuous frames.
461 while (!continuous_frames.empty()) {
462 auto frame = continuous_frames.front();
463 continuous_frames.pop();
464
Ilya Nikolaevskiy6551faf2019-01-10 15:16:47 +0100465 if (!last_continuous_frame_ || *last_continuous_frame_ < frame->first) {
466 last_continuous_frame_ = frame->first;
467 }
philipele0b2f152016-09-28 10:23:49 +0200468
469 // Loop through all dependent frames, and if that frame no longer has
470 // any unfulfilled dependencies then that frame is continuous as well.
Elad Alon69321dd2019-01-10 15:02:54 +0100471 for (size_t d = 0; d < frame->second.dependent_frames.size(); ++d) {
philipele0b2f152016-09-28 10:23:49 +0200472 auto frame_ref = frames_.find(frame->second.dependent_frames[d]);
philipel112adf92017-06-15 09:06:21 -0700473 RTC_DCHECK(frame_ref != frames_.end());
philipele0b2f152016-09-28 10:23:49 +0200474
philipel112adf92017-06-15 09:06:21 -0700475 // TODO(philipel): Look into why we've seen this happen.
476 if (frame_ref != frames_.end()) {
477 --frame_ref->second.num_missing_continuous;
478 if (frame_ref->second.num_missing_continuous == 0) {
479 frame_ref->second.continuous = true;
480 continuous_frames.push(frame_ref);
481 }
philipele0b2f152016-09-28 10:23:49 +0200482 }
483 }
484 }
485}
486
487void FrameBuffer::PropagateDecodability(const FrameInfo& info) {
tommidb23ea62017-03-03 07:21:18 -0800488 TRACE_EVENT0("webrtc", "FrameBuffer::PropagateDecodability");
Elad Alon69321dd2019-01-10 15:02:54 +0100489 for (size_t d = 0; d < info.dependent_frames.size(); ++d) {
philipele0b2f152016-09-28 10:23:49 +0200490 auto ref_info = frames_.find(info.dependent_frames[d]);
philipel93e451b2016-10-06 12:25:13 +0200491 RTC_DCHECK(ref_info != frames_.end());
tommie95b78b2017-05-14 07:23:11 -0700492 // TODO(philipel): Look into why we've seen this happen.
493 if (ref_info != frames_.end()) {
494 RTC_DCHECK_GT(ref_info->second.num_missing_decodable, 0U);
495 --ref_info->second.num_missing_decodable;
496 }
philipele0b2f152016-09-28 10:23:49 +0200497 }
498}
499
philipele7c891f2018-02-22 14:35:06 +0100500bool FrameBuffer::UpdateFrameInfoWithIncomingFrame(const EncodedFrame& frame,
philipele0b2f152016-09-28 10:23:49 +0200501 FrameMap::iterator info) {
tommidb23ea62017-03-03 07:21:18 -0800502 TRACE_EVENT0("webrtc", "FrameBuffer::UpdateFrameInfoWithIncomingFrame");
philipel0fa82a62018-03-19 15:34:53 +0100503 const VideoLayerFrameId& id = frame.id;
philipele0b2f152016-09-28 10:23:49 +0200504
Ilya Nikolaevskiy13717842019-01-14 13:24:22 +0100505 auto last_decoded_frame = decoded_frames_history_.GetLastDecodedFrameId();
506 RTC_DCHECK(!last_decoded_frame || *last_decoded_frame < info->first);
philipele0b2f152016-09-28 10:23:49 +0200507
philipel798b2822018-06-11 13:10:14 +0200508 // In this function we determine how many missing dependencies this |frame|
509 // has to become continuous/decodable. If a frame that this |frame| depend
510 // on has already been decoded then we can ignore that dependency since it has
511 // already been fulfilled.
512 //
513 // For all other frames we will register a backwards reference to this |frame|
514 // so that |num_missing_continuous| and |num_missing_decodable| can be
515 // decremented as frames become continuous/are decoded.
516 struct Dependency {
517 VideoLayerFrameId id;
518 bool continuous;
519 };
520 std::vector<Dependency> not_yet_fulfilled_dependencies;
521
522 // Find all dependencies that have not yet been fulfilled.
philipele0b2f152016-09-28 10:23:49 +0200523 for (size_t i = 0; i < frame.num_references; ++i) {
philipel0fa82a62018-03-19 15:34:53 +0100524 VideoLayerFrameId ref_key(frame.references[i], frame.id.spatial_layer);
philipel798b2822018-06-11 13:10:14 +0200525 // Does |frame| depend on a frame earlier than the last decoded one?
Ilya Nikolaevskiy13717842019-01-14 13:24:22 +0100526 if (last_decoded_frame && ref_key <= *last_decoded_frame) {
philipel798b2822018-06-11 13:10:14 +0200527 // Was that frame decoded? If not, this |frame| will never become
528 // decodable.
Ilya Nikolaevskiy13717842019-01-14 13:24:22 +0100529 if (!decoded_frames_history_.WasDecoded(ref_key)) {
philipel65e1f942017-07-24 08:26:53 -0700530 int64_t now_ms = clock_->TimeInMilliseconds();
531 if (last_log_non_decoded_ms_ + kLogNonDecodedIntervalMs < now_ms) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100532 RTC_LOG(LS_WARNING)
philipel0fa82a62018-03-19 15:34:53 +0100533 << "Frame with (picture_id:spatial_id) (" << id.picture_id << ":"
534 << static_cast<int>(id.spatial_layer)
philipel65e1f942017-07-24 08:26:53 -0700535 << ") depends on a non-decoded frame more previous than"
536 << " the last decoded frame, dropping frame.";
537 last_log_non_decoded_ms_ = now_ms;
538 }
philipele0b2f152016-09-28 10:23:49 +0200539 return false;
540 }
philipele0b2f152016-09-28 10:23:49 +0200541 } else {
Ilya Nikolaevskiy6551faf2019-01-10 15:16:47 +0100542 auto ref_info = frames_.find(ref_key);
philipel798b2822018-06-11 13:10:14 +0200543 bool ref_continuous =
544 ref_info != frames_.end() && ref_info->second.continuous;
545 not_yet_fulfilled_dependencies.push_back({ref_key, ref_continuous});
philipele0b2f152016-09-28 10:23:49 +0200546 }
philipelbe7a9e52016-05-19 12:19:35 +0200547 }
548
philipel798b2822018-06-11 13:10:14 +0200549 // Does |frame| depend on the lower spatial layer?
philipelbe7a9e52016-05-19 12:19:35 +0200550 if (frame.inter_layer_predicted) {
philipel0fa82a62018-03-19 15:34:53 +0100551 VideoLayerFrameId ref_key(frame.id.picture_id, frame.id.spatial_layer - 1);
philipel798b2822018-06-11 13:10:14 +0200552 auto ref_info = frames_.find(ref_key);
philipele0b2f152016-09-28 10:23:49 +0200553
Ilya Nikolaevskiy6551faf2019-01-10 15:16:47 +0100554 bool lower_layer_decoded =
Ilya Nikolaevskiy13717842019-01-14 13:24:22 +0100555 last_decoded_frame && *last_decoded_frame == ref_key;
philipel798b2822018-06-11 13:10:14 +0200556 bool lower_layer_continuous =
Ilya Nikolaevskiy6551faf2019-01-10 15:16:47 +0100557 lower_layer_decoded ||
558 (ref_info != frames_.end() && ref_info->second.continuous);
philipel798b2822018-06-11 13:10:14 +0200559
560 if (!lower_layer_continuous || !lower_layer_decoded) {
561 not_yet_fulfilled_dependencies.push_back(
562 {ref_key, lower_layer_continuous});
philipele0b2f152016-09-28 10:23:49 +0200563 }
philipelbe7a9e52016-05-19 12:19:35 +0200564 }
565
philipel798b2822018-06-11 13:10:14 +0200566 info->second.num_missing_continuous = not_yet_fulfilled_dependencies.size();
567 info->second.num_missing_decodable = not_yet_fulfilled_dependencies.size();
568
569 for (const Dependency& dep : not_yet_fulfilled_dependencies) {
570 if (dep.continuous)
571 --info->second.num_missing_continuous;
572
Elad Alon69321dd2019-01-10 15:02:54 +0100573 frames_[dep.id].dependent_frames.push_back(id);
philipel798b2822018-06-11 13:10:14 +0200574 }
philipel93e451b2016-10-06 12:25:13 +0200575
philipelbe7a9e52016-05-19 12:19:35 +0200576 return true;
577}
578
philipelbe742702016-11-30 01:31:40 -0800579void FrameBuffer::UpdateJitterDelay() {
tommidb23ea62017-03-03 07:21:18 -0800580 TRACE_EVENT0("webrtc", "FrameBuffer::UpdateJitterDelay");
philipela45102f2017-02-22 05:30:39 -0800581 if (!stats_callback_)
582 return;
philipelbe742702016-11-30 01:31:40 -0800583
philipela45102f2017-02-22 05:30:39 -0800584 int decode_ms;
585 int max_decode_ms;
586 int current_delay_ms;
587 int target_delay_ms;
588 int jitter_buffer_ms;
589 int min_playout_delay_ms;
590 int render_delay_ms;
591 if (timing_->GetTimings(&decode_ms, &max_decode_ms, &current_delay_ms,
592 &target_delay_ms, &jitter_buffer_ms,
593 &min_playout_delay_ms, &render_delay_ms)) {
594 stats_callback_->OnFrameBufferTimingsUpdated(
595 decode_ms, max_decode_ms, current_delay_ms, target_delay_ms,
596 jitter_buffer_ms, min_playout_delay_ms, render_delay_ms);
philipelbe742702016-11-30 01:31:40 -0800597 }
philipel266f0a42016-11-28 08:49:07 -0800598}
599
ilnik2edc6842017-07-06 03:06:50 -0700600void FrameBuffer::UpdateTimingFrameInfo() {
601 TRACE_EVENT0("webrtc", "FrameBuffer::UpdateTimingFrameInfo");
Danil Chapovalov0040b662018-06-18 10:48:16 +0200602 absl::optional<TimingFrameInfo> info = timing_->GetTimingFrameInfo();
philipel97187112018-03-23 10:43:21 +0100603 if (info && stats_callback_)
ilnik2edc6842017-07-06 03:06:50 -0700604 stats_callback_->OnTimingFrameInfoUpdated(*info);
605}
606
philipelfcc60062017-01-18 05:35:20 -0800607void FrameBuffer::ClearFramesAndHistory() {
ilnik2edc6842017-07-06 03:06:50 -0700608 TRACE_EVENT0("webrtc", "FrameBuffer::ClearFramesAndHistory");
philipelfcc60062017-01-18 05:35:20 -0800609 frames_.clear();
Ilya Nikolaevskiy6551faf2019-01-10 15:16:47 +0100610 last_continuous_frame_.reset();
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100611 frames_to_decode_.clear();
Ilya Nikolaevskiy13717842019-01-14 13:24:22 +0100612 decoded_frames_history_.Clear();
philipelfcc60062017-01-18 05:35:20 -0800613}
614
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100615EncodedFrame* FrameBuffer::CombineAndDeleteFrames(
616 const std::vector<EncodedFrame*>& frames) const {
617 RTC_DCHECK(!frames.empty());
Sergey Silkin61832dd2018-12-20 14:32:14 +0100618 EncodedFrame* first_frame = frames[0];
619 EncodedFrame* last_frame = frames.back();
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100620 size_t total_length = 0;
621 for (size_t i = 0; i < frames.size(); ++i) {
622 total_length += frames[i]->size();
623 }
Sergey Silkin61832dd2018-12-20 14:32:14 +0100624 first_frame->VerifyAndAllocate(total_length);
625
626 // Spatial index of combined frame is set equal to spatial index of its top
627 // spatial layer.
628 first_frame->SetSpatialIndex(last_frame->id.spatial_layer);
629 first_frame->id.spatial_layer = last_frame->id.spatial_layer;
630
631 first_frame->video_timing_mutable()->network2_timestamp_ms =
632 last_frame->video_timing().network2_timestamp_ms;
633 first_frame->video_timing_mutable()->receive_finish_ms =
634 last_frame->video_timing().receive_finish_ms;
635
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100636 // Append all remaining frames to the first one.
Niels Möller9c843902019-01-11 10:21:35 +0100637 uint8_t* buffer = first_frame->data() + first_frame->size();
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100638 for (size_t i = 1; i < frames.size(); ++i) {
Sergey Silkin61832dd2018-12-20 14:32:14 +0100639 EncodedFrame* next_frame = frames[i];
Niels Möller9c843902019-01-11 10:21:35 +0100640 memcpy(buffer, next_frame->data(), next_frame->size());
Sergey Silkin61832dd2018-12-20 14:32:14 +0100641 buffer += next_frame->size();
642 delete next_frame;
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100643 }
Sergey Silkin61832dd2018-12-20 14:32:14 +0100644 first_frame->set_size(total_length);
645 return first_frame;
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100646}
647
Niels Möllerbe682d42018-03-27 08:31:45 +0200648FrameBuffer::FrameInfo::FrameInfo() = default;
649FrameBuffer::FrameInfo::FrameInfo(FrameInfo&&) = default;
650FrameBuffer::FrameInfo::~FrameInfo() = default;
651
philipelbe7a9e52016-05-19 12:19:35 +0200652} // namespace video_coding
653} // namespace webrtc