blob: b4b9c117b96a2894b06fc23540dd09401a77d9a3 [file] [log] [blame]
philipelbe7a9e52016-05-19 12:19:35 +02001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "modules/video_coding/frame_buffer2.h"
philipelbe7a9e52016-05-19 12:19:35 +020012
13#include <algorithm>
Yves Gerey3e707812018-11-28 16:47:49 +010014#include <cstdlib>
15#include <iterator>
Evan Shrubsole0072c212021-11-10 11:41:14 +010016#include <memory>
philipele0b2f152016-09-28 10:23:49 +020017#include <queue>
Yves Gerey3e707812018-11-28 16:47:49 +010018#include <utility>
philipel798b2822018-06-11 13:10:14 +020019#include <vector>
philipelbe7a9e52016-05-19 12:19:35 +020020
Evan Shrubsolef83d4262021-12-15 15:05:15 +010021#include "absl/container/inlined_vector.h"
Evan Shrubsole13e42a82022-03-07 13:21:51 +010022#include "api/units/data_size.h"
23#include "api/units/time_delta.h"
Yves Gerey3e707812018-11-28 16:47:49 +010024#include "api/video/encoded_image.h"
25#include "api/video/video_timing.h"
Evan Shrubsolef83d4262021-12-15 15:05:15 +010026#include "modules/video_coding/frame_helpers.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020027#include "modules/video_coding/include/video_coding_defines.h"
28#include "modules/video_coding/jitter_estimator.h"
29#include "modules/video_coding/timing.h"
30#include "rtc_base/checks.h"
Yves Gerey3e707812018-11-28 16:47:49 +010031#include "rtc_base/experiments/rtt_mult_experiment.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020032#include "rtc_base/logging.h"
Yves Gerey3e707812018-11-28 16:47:49 +010033#include "rtc_base/numerics/sequence_number_util.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020034#include "rtc_base/trace_event.h"
35#include "system_wrappers/include/clock.h"
philipel707f2782017-10-02 14:10:28 +020036#include "system_wrappers/include/field_trial.h"
philipelbe7a9e52016-05-19 12:19:35 +020037
38namespace webrtc {
39namespace video_coding {
40
41namespace {
philipele0b2f152016-09-28 10:23:49 +020042// Max number of frames the buffer will hold.
Ilya Nikolaevskiy13717842019-01-14 13:24:22 +010043constexpr size_t kMaxFramesBuffered = 800;
philipelbe7a9e52016-05-19 12:19:35 +020044
Johannes Kron23bfff32021-09-28 21:31:46 +020045// Default value for the maximum decode queue size that is used when the
46// low-latency renderer is used.
47constexpr size_t kZeroPlayoutDelayDefaultMaxDecodeQueueSize = 8;
48
philipele0b2f152016-09-28 10:23:49 +020049// Max number of decoded frame info that will be saved.
Ilya Nikolaevskiy13717842019-01-14 13:24:22 +010050constexpr int kMaxFramesHistory = 1 << 13;
philipel65e1f942017-07-24 08:26:53 -070051
Ilya Nikolaevskiy8c4fe162018-02-27 15:49:47 +010052// The time it's allowed for a frame to be late to its rendering prediction and
53// still be rendered.
Ilya Nikolaevskiy7eef0072018-02-28 09:59:26 +010054constexpr int kMaxAllowedFrameDelayMs = 5;
Ilya Nikolaevskiy8c4fe162018-02-27 15:49:47 +010055
philipel65e1f942017-07-24 08:26:53 -070056constexpr int64_t kLogNonDecodedIntervalMs = 5000;
philipelbe7a9e52016-05-19 12:19:35 +020057} // namespace
58
philipelbe7a9e52016-05-19 12:19:35 +020059FrameBuffer::FrameBuffer(Clock* clock,
philipela45102f2017-02-22 05:30:39 -080060 VCMTiming* timing,
61 VCMReceiveStatisticsCallback* stats_callback)
Ilya Nikolaevskiy13717842019-01-14 13:24:22 +010062 : decoded_frames_history_(kMaxFramesHistory),
63 clock_(clock),
Sebastian Jansson11d0d7b2019-04-11 12:39:34 +020064 callback_queue_(nullptr),
Niels Möllerd9c2d942019-04-30 09:16:36 +020065 jitter_estimator_(clock),
philipelbe7a9e52016-05-19 12:19:35 +020066 timing_(timing),
philipel29f730e2017-03-15 08:10:08 -070067 stopped_(false),
philipela45102f2017-02-22 05:30:39 -080068 protection_mode_(kProtectionNack),
philipel65e1f942017-07-24 08:26:53 -070069 stats_callback_(stats_callback),
Elad Alone4b50232019-01-14 18:56:14 +010070 last_log_non_decoded_ms_(-kLogNonDecodedIntervalMs),
Johannes Kron2ddc39e2021-08-10 16:56:12 +020071 rtt_mult_settings_(RttMultExperiment::GetRttMultValue()),
Johannes Kron23bfff32021-09-28 21:31:46 +020072 zero_playout_delay_max_decode_queue_size_(
73 "max_decode_queue_size",
74 kZeroPlayoutDelayDefaultMaxDecodeQueueSize) {
Johannes Kron2ddc39e2021-08-10 16:56:12 +020075 ParseFieldTrial({&zero_playout_delay_max_decode_queue_size_},
76 field_trial::FindFullName("WebRTC-ZeroPlayoutDelay"));
Tommi430951a2020-05-19 23:27:29 +020077 callback_checker_.Detach();
78}
philipel266f0a42016-11-28 08:49:07 -080079
Tommi430951a2020-05-19 23:27:29 +020080FrameBuffer::~FrameBuffer() {
81 RTC_DCHECK_RUN_ON(&construction_checker_);
82}
philipelbe7a9e52016-05-19 12:19:35 +020083
Evan Shrubsole3d29efd2021-12-07 14:11:45 +010084void FrameBuffer::NextFrame(int64_t max_wait_time_ms,
85 bool keyframe_required,
86 rtc::TaskQueue* callback_queue,
87 NextFrameCallback handler) {
Tommi430951a2020-05-19 23:27:29 +020088 RTC_DCHECK_RUN_ON(&callback_checker_);
89 RTC_DCHECK(callback_queue->IsCurrent());
Sebastian Jansson11d0d7b2019-04-11 12:39:34 +020090 TRACE_EVENT0("webrtc", "FrameBuffer::NextFrame");
91 int64_t latest_return_time_ms =
92 clock_->TimeInMilliseconds() + max_wait_time_ms;
Tommi430951a2020-05-19 23:27:29 +020093
Markus Handell6deec382020-07-07 12:17:12 +020094 MutexLock lock(&mutex_);
Sebastian Jansson11d0d7b2019-04-11 12:39:34 +020095 if (stopped_) {
96 return;
97 }
98 latest_return_time_ms_ = latest_return_time_ms;
99 keyframe_required_ = keyframe_required;
100 frame_handler_ = handler;
101 callback_queue_ = callback_queue;
102 StartWaitForNextFrameOnQueue();
103}
104
105void FrameBuffer::StartWaitForNextFrameOnQueue() {
106 RTC_DCHECK(callback_queue_);
107 RTC_DCHECK(!callback_task_.Running());
Evan Shrubsoled6cdf802022-03-02 15:13:55 +0100108 int64_t wait_ms = FindNextFrame(clock_->CurrentTime());
Sebastian Jansson11d0d7b2019-04-11 12:39:34 +0200109 callback_task_ = RepeatingTaskHandle::DelayedStart(
Henrik Boström4c72f992022-01-25 08:15:50 +0100110 callback_queue_->Get(), TimeDelta::Millis(wait_ms),
111 [this] {
Tommi430951a2020-05-19 23:27:29 +0200112 RTC_DCHECK_RUN_ON(&callback_checker_);
Sebastian Jansson11d0d7b2019-04-11 12:39:34 +0200113 // If this task has not been cancelled, we did not get any new frames
114 // while waiting. Continue with frame delivery.
Markus Handell334b1fd2020-12-14 00:08:11 +0100115 std::unique_ptr<EncodedFrame> frame;
Evan Shrubsole3d29efd2021-12-07 14:11:45 +0100116 NextFrameCallback frame_handler;
Markus Handell334b1fd2020-12-14 00:08:11 +0100117 {
118 MutexLock lock(&mutex_);
119 if (!frames_to_decode_.empty()) {
120 // We have frames, deliver!
Evan Shrubsole0072c212021-11-10 11:41:14 +0100121 frame = GetNextFrame();
Evan Shrubsoled6cdf802022-03-02 15:13:55 +0100122 timing_->SetLastDecodeScheduledTimestamp(clock_->CurrentTime());
Markus Handell334b1fd2020-12-14 00:08:11 +0100123 } else if (clock_->TimeInMilliseconds() < latest_return_time_ms_) {
124 // If there's no frames to decode and there is still time left, it
125 // means that the frame buffer was cleared between creation and
126 // execution of this task. Continue waiting for the remaining time.
Evan Shrubsoled6cdf802022-03-02 15:13:55 +0100127 int64_t wait_ms = FindNextFrame(clock_->CurrentTime());
Markus Handell334b1fd2020-12-14 00:08:11 +0100128 return TimeDelta::Millis(wait_ms);
129 }
130 frame_handler = std::move(frame_handler_);
Sebastian Jansson11d0d7b2019-04-11 12:39:34 +0200131 CancelCallback();
Sebastian Jansson11d0d7b2019-04-11 12:39:34 +0200132 }
Markus Handell334b1fd2020-12-14 00:08:11 +0100133 // Deliver frame, if any. Otherwise signal timeout.
Evan Shrubsole3d29efd2021-12-07 14:11:45 +0100134 frame_handler(std::move(frame));
Markus Handell334b1fd2020-12-14 00:08:11 +0100135 return TimeDelta::Zero(); // Ignored.
Henrik Boström4c72f992022-01-25 08:15:50 +0100136 },
137 TaskQueueBase::DelayPrecision::kHigh);
Sebastian Jansson11d0d7b2019-04-11 12:39:34 +0200138}
139
Evan Shrubsoled6cdf802022-03-02 15:13:55 +0100140int64_t FrameBuffer::FindNextFrame(Timestamp now) {
141 int64_t wait_ms = latest_return_time_ms_ - now.ms();
Sebastian Jansson1c747f52019-04-04 13:01:39 +0200142 frames_to_decode_.clear();
143
Artem Titovdcd7fc72021-08-09 13:02:57 +0200144 // `last_continuous_frame_` may be empty below, but nullopt is smaller
Sebastian Jansson1c747f52019-04-04 13:01:39 +0200145 // than everything else and loop will immediately terminate as expected.
146 for (auto frame_it = frames_.begin();
147 frame_it != frames_.end() && frame_it->first <= last_continuous_frame_;
148 ++frame_it) {
149 if (!frame_it->second.continuous ||
150 frame_it->second.num_missing_decodable > 0) {
151 continue;
152 }
153
154 EncodedFrame* frame = frame_it->second.frame.get();
155
156 if (keyframe_required_ && !frame->is_keyframe())
157 continue;
158
159 auto last_decoded_frame_timestamp =
160 decoded_frames_history_.GetLastDecodedFrameTimestamp();
161
162 // TODO(https://bugs.webrtc.org/9974): consider removing this check
163 // as it may make a stream undecodable after a very long delay between
164 // frames.
165 if (last_decoded_frame_timestamp &&
166 AheadOf(*last_decoded_frame_timestamp, frame->Timestamp())) {
167 continue;
168 }
169
Sebastian Jansson1c747f52019-04-04 13:01:39 +0200170 // Gather all remaining frames for the same superframe.
171 std::vector<FrameMap::iterator> current_superframe;
172 current_superframe.push_back(frame_it);
173 bool last_layer_completed = frame_it->second.frame->is_last_spatial_layer;
174 FrameMap::iterator next_frame_it = frame_it;
philipela65d7852020-11-20 17:49:24 +0100175 while (!last_layer_completed) {
Sebastian Jansson1c747f52019-04-04 13:01:39 +0200176 ++next_frame_it;
philipela65d7852020-11-20 17:49:24 +0100177
178 if (next_frame_it == frames_.end() || !next_frame_it->second.frame) {
179 break;
180 }
181
182 if (next_frame_it->second.frame->Timestamp() != frame->Timestamp() ||
Sebastian Jansson1c747f52019-04-04 13:01:39 +0200183 !next_frame_it->second.continuous) {
184 break;
185 }
philipela65d7852020-11-20 17:49:24 +0100186
187 if (next_frame_it->second.num_missing_decodable > 0) {
philipelcb327d92020-12-10 10:49:20 +0100188 bool has_inter_layer_dependency = false;
189 for (size_t i = 0; i < EncodedFrame::kMaxFrameReferences &&
philipela65d7852020-11-20 17:49:24 +0100190 i < next_frame_it->second.frame->num_references;
191 ++i) {
philipel9aa9b8d2021-02-15 13:31:29 +0100192 if (next_frame_it->second.frame->references[i] >= frame_it->first) {
philipela65d7852020-11-20 17:49:24 +0100193 has_inter_layer_dependency = true;
philipelcb327d92020-12-10 10:49:20 +0100194 break;
philipela65d7852020-11-20 17:49:24 +0100195 }
196 }
197
198 // If the frame has an undecoded dependency that is not within the same
philipelcb327d92020-12-10 10:49:20 +0100199 // temporal unit then this frame is not yet ready to be decoded. If it
philipela65d7852020-11-20 17:49:24 +0100200 // is within the same temporal unit then the not yet decoded dependency
201 // is just a lower spatial frame, which is ok.
202 if (!has_inter_layer_dependency ||
203 next_frame_it->second.num_missing_decodable > 1) {
204 break;
205 }
Sebastian Jansson1c747f52019-04-04 13:01:39 +0200206 }
philipela65d7852020-11-20 17:49:24 +0100207
Sebastian Jansson1c747f52019-04-04 13:01:39 +0200208 current_superframe.push_back(next_frame_it);
209 last_layer_completed = next_frame_it->second.frame->is_last_spatial_layer;
210 }
211 // Check if the current superframe is complete.
212 // TODO(bugs.webrtc.org/10064): consider returning all available to
213 // decode frames even if the superframe is not complete yet.
214 if (!last_layer_completed) {
215 continue;
216 }
217
218 frames_to_decode_ = std::move(current_superframe);
219
Evan Shrubsoled6cdf802022-03-02 15:13:55 +0100220 absl::optional<Timestamp> render_time = frame->RenderTimestamp();
221 if (!render_time) {
222 render_time = timing_->RenderTime(frame->Timestamp(), now);
223 frame->SetRenderTime(render_time->ms());
Sebastian Jansson1c747f52019-04-04 13:01:39 +0200224 }
Johannes Kron2ddc39e2021-08-10 16:56:12 +0200225 bool too_many_frames_queued =
226 frames_.size() > zero_playout_delay_max_decode_queue_size_ ? true
227 : false;
Evan Shrubsoled6cdf802022-03-02 15:13:55 +0100228 wait_ms =
229 timing_->MaxWaitingTime(*render_time, now, too_many_frames_queued).ms();
Sebastian Jansson1c747f52019-04-04 13:01:39 +0200230
231 // This will cause the frame buffer to prefer high framerate rather
232 // than high resolution in the case of the decoder not decoding fast
233 // enough and the stream has multiple spatial and temporal layers.
234 // For multiple temporal layers it may cause non-base layer frames to be
235 // skipped if they are late.
236 if (wait_ms < -kMaxAllowedFrameDelayMs)
237 continue;
238
239 break;
240 }
Evan Shrubsoled6cdf802022-03-02 15:13:55 +0100241 wait_ms = std::min<int64_t>(wait_ms, latest_return_time_ms_ - now.ms());
Sebastian Jansson1c747f52019-04-04 13:01:39 +0200242 wait_ms = std::max<int64_t>(wait_ms, 0);
243 return wait_ms;
244}
245
Evan Shrubsole0072c212021-11-10 11:41:14 +0100246std::unique_ptr<EncodedFrame> FrameBuffer::GetNextFrame() {
Tommi430951a2020-05-19 23:27:29 +0200247 RTC_DCHECK_RUN_ON(&callback_checker_);
Evan Shrubsoled6cdf802022-03-02 15:13:55 +0100248 Timestamp now = clock_->CurrentTime();
Artem Titovdcd7fc72021-08-09 13:02:57 +0200249 // TODO(ilnik): remove `frames_out` use frames_to_decode_ directly.
Evan Shrubsole0072c212021-11-10 11:41:14 +0100250 std::vector<std::unique_ptr<EncodedFrame>> frames_out;
Sebastian Jansson1c747f52019-04-04 13:01:39 +0200251
252 RTC_DCHECK(!frames_to_decode_.empty());
253 bool superframe_delayed_by_retransmission = false;
Evan Shrubsole13e42a82022-03-07 13:21:51 +0100254 DataSize superframe_size = DataSize::Zero();
Evan Shrubsole5b9b9aa2021-11-01 10:28:31 +0100255 const EncodedFrame& first_frame = *frames_to_decode_[0]->second.frame;
Evan Shrubsoled6cdf802022-03-02 15:13:55 +0100256 absl::optional<Timestamp> render_time = first_frame.RenderTimestamp();
Evan Shrubsole5b9b9aa2021-11-01 10:28:31 +0100257 int64_t receive_time_ms = first_frame.ReceivedTime();
Sebastian Jansson1c747f52019-04-04 13:01:39 +0200258 // Gracefully handle bad RTP timestamps and render time issues.
Evan Shrubsoled6cdf802022-03-02 15:13:55 +0100259 if (!render_time ||
260 FrameHasBadRenderTiming(*render_time, now, timing_->TargetVideoDelay())) {
Niels Möllerd9c2d942019-04-30 09:16:36 +0200261 jitter_estimator_.Reset();
Sebastian Jansson1c747f52019-04-04 13:01:39 +0200262 timing_->Reset();
Evan Shrubsoled6cdf802022-03-02 15:13:55 +0100263 render_time = timing_->RenderTime(first_frame.Timestamp(), now);
Sebastian Jansson1c747f52019-04-04 13:01:39 +0200264 }
265
266 for (FrameMap::iterator& frame_it : frames_to_decode_) {
267 RTC_DCHECK(frame_it != frames_.end());
Evan Shrubsole0072c212021-11-10 11:41:14 +0100268 std::unique_ptr<EncodedFrame> frame = std::move(frame_it->second.frame);
Sebastian Jansson1c747f52019-04-04 13:01:39 +0200269
Evan Shrubsoled6cdf802022-03-02 15:13:55 +0100270 frame->SetRenderTime(render_time->ms());
Sebastian Jansson1c747f52019-04-04 13:01:39 +0200271
272 superframe_delayed_by_retransmission |= frame->delayed_by_retransmission();
273 receive_time_ms = std::max(receive_time_ms, frame->ReceivedTime());
Evan Shrubsole13e42a82022-03-07 13:21:51 +0100274 superframe_size += DataSize::Bytes(frame->size());
Sebastian Jansson1c747f52019-04-04 13:01:39 +0200275
276 PropagateDecodability(frame_it->second);
277 decoded_frames_history_.InsertDecoded(frame_it->first, frame->Timestamp());
278
279 // Remove decoded frame and all undecoded frames before it.
Johannes Kron0c141c52019-08-26 15:04:43 +0200280 if (stats_callback_) {
philipel9aa9b8d2021-02-15 13:31:29 +0100281 unsigned int dropped_frames =
282 std::count_if(frames_.begin(), frame_it,
283 [](const std::pair<const int64_t, FrameInfo>& frame) {
284 return frame.second.frame != nullptr;
285 });
Johannes Kron0c141c52019-08-26 15:04:43 +0200286 if (dropped_frames > 0) {
287 stats_callback_->OnDroppedFrames(dropped_frames);
288 }
289 }
290
Sebastian Jansson1c747f52019-04-04 13:01:39 +0200291 frames_.erase(frames_.begin(), ++frame_it);
292
Evan Shrubsole0072c212021-11-10 11:41:14 +0100293 frames_out.emplace_back(std::move(frame));
Sebastian Jansson1c747f52019-04-04 13:01:39 +0200294 }
295
296 if (!superframe_delayed_by_retransmission) {
Evan Shrubsolee9126c12022-03-07 14:50:51 +0100297 auto frame_delay = inter_frame_delay_.CalculateDelay(
298 first_frame.Timestamp(), Timestamp::Millis(receive_time_ms));
Sebastian Jansson1c747f52019-04-04 13:01:39 +0200299
Evan Shrubsolee9126c12022-03-07 14:50:51 +0100300 if (frame_delay) {
301 jitter_estimator_.UpdateEstimate(*frame_delay, superframe_size);
Sebastian Jansson1c747f52019-04-04 13:01:39 +0200302 }
303
304 float rtt_mult = protection_mode_ == kProtectionNackFEC ? 0.0 : 1.0;
Evan Shrubsole13e42a82022-03-07 13:21:51 +0100305 absl::optional<TimeDelta> rtt_mult_add_cap_ms = absl::nullopt;
“Michaeld3a4ebe2019-06-07 03:55:01 -0500306 if (rtt_mult_settings_.has_value()) {
307 rtt_mult = rtt_mult_settings_->rtt_mult_setting;
Evan Shrubsole13e42a82022-03-07 13:21:51 +0100308 rtt_mult_add_cap_ms =
309 TimeDelta::Millis(rtt_mult_settings_->rtt_mult_add_cap_ms);
Sebastian Jansson1c747f52019-04-04 13:01:39 +0200310 }
Evan Shrubsole13e42a82022-03-07 13:21:51 +0100311 timing_->SetJitterDelay(
312 jitter_estimator_.GetJitterEstimate(rtt_mult, rtt_mult_add_cap_ms));
Evan Shrubsoled6cdf802022-03-02 15:13:55 +0100313 timing_->UpdateCurrentDelay(*render_time, now);
Sebastian Jansson1c747f52019-04-04 13:01:39 +0200314 } else {
Evan Shrubsole9146d762021-10-13 11:19:48 +0200315 if (RttMultExperiment::RttMultEnabled())
Niels Möllerd9c2d942019-04-30 09:16:36 +0200316 jitter_estimator_.FrameNacked();
Sebastian Jansson1c747f52019-04-04 13:01:39 +0200317 }
318
319 UpdateJitterDelay();
320 UpdateTimingFrameInfo();
321
322 if (frames_out.size() == 1) {
Evan Shrubsole0072c212021-11-10 11:41:14 +0100323 return std::move(frames_out[0]);
Sebastian Jansson1c747f52019-04-04 13:01:39 +0200324 } else {
Evan Shrubsole0072c212021-11-10 11:41:14 +0100325 return CombineAndDeleteFrames(std::move(frames_out));
Sebastian Jansson1c747f52019-04-04 13:01:39 +0200326 }
327}
328
philipel4f6cd6a2016-08-03 10:59:32 +0200329void FrameBuffer::SetProtectionMode(VCMVideoProtection mode) {
tommidb23ea62017-03-03 07:21:18 -0800330 TRACE_EVENT0("webrtc", "FrameBuffer::SetProtectionMode");
Markus Handell6deec382020-07-07 12:17:12 +0200331 MutexLock lock(&mutex_);
Henrik Boströmc680c4a2019-04-03 10:27:36 +0000332 protection_mode_ = mode;
philipel4f6cd6a2016-08-03 10:59:32 +0200333}
334
philipel504c47d2016-06-30 17:33:02 +0200335void FrameBuffer::Stop() {
tommidb23ea62017-03-03 07:21:18 -0800336 TRACE_EVENT0("webrtc", "FrameBuffer::Stop");
Markus Handell6deec382020-07-07 12:17:12 +0200337 MutexLock lock(&mutex_);
Tommi430951a2020-05-19 23:27:29 +0200338 if (stopped_)
339 return;
Henrik Boströmc680c4a2019-04-03 10:27:36 +0000340 stopped_ = true;
Tommi430951a2020-05-19 23:27:29 +0200341
Sebastian Jansson11d0d7b2019-04-11 12:39:34 +0200342 CancelCallback();
philipel504c47d2016-06-30 17:33:02 +0200343}
344
Ilya Nikolaevskiye6a2d942018-11-07 14:32:28 +0100345void FrameBuffer::Clear() {
Markus Handell6deec382020-07-07 12:17:12 +0200346 MutexLock lock(&mutex_);
Henrik Boströmc680c4a2019-04-03 10:27:36 +0000347 ClearFramesAndHistory();
Ilya Nikolaevskiye6a2d942018-11-07 14:32:28 +0100348}
349
Johannes Kron111e9812020-10-26 13:54:40 +0100350int FrameBuffer::Size() {
351 MutexLock lock(&mutex_);
352 return frames_.size();
353}
354
philipele21be1d2017-09-25 06:37:12 -0700355void FrameBuffer::UpdateRtt(int64_t rtt_ms) {
Markus Handell6deec382020-07-07 12:17:12 +0200356 MutexLock lock(&mutex_);
Evan Shrubsole13e42a82022-03-07 13:21:51 +0100357 jitter_estimator_.UpdateRtt(TimeDelta::Millis(rtt_ms));
philipele21be1d2017-09-25 06:37:12 -0700358}
359
philipele7c891f2018-02-22 14:35:06 +0100360bool FrameBuffer::ValidReferences(const EncodedFrame& frame) const {
philipel112adf92017-06-15 09:06:21 -0700361 for (size_t i = 0; i < frame.num_references; ++i) {
philipel9aa9b8d2021-02-15 13:31:29 +0100362 if (frame.references[i] >= frame.Id())
philipel112adf92017-06-15 09:06:21 -0700363 return false;
philipel3b3c9c42017-09-11 09:38:36 -0700364
philipel112adf92017-06-15 09:06:21 -0700365 for (size_t j = i + 1; j < frame.num_references; ++j) {
366 if (frame.references[i] == frame.references[j])
367 return false;
368 }
369 }
370
philipel112adf92017-06-15 09:06:21 -0700371 return true;
372}
373
Sebastian Jansson11d0d7b2019-04-11 12:39:34 +0200374void FrameBuffer::CancelCallback() {
Tommi430951a2020-05-19 23:27:29 +0200375 // Called from the callback queue or from within Stop().
Sebastian Jansson11d0d7b2019-04-11 12:39:34 +0200376 frame_handler_ = {};
377 callback_task_.Stop();
378 callback_queue_ = nullptr;
Tommi430951a2020-05-19 23:27:29 +0200379 callback_checker_.Detach();
Sebastian Jansson11d0d7b2019-04-11 12:39:34 +0200380}
381
philipele7c891f2018-02-22 14:35:06 +0100382int64_t FrameBuffer::InsertFrame(std::unique_ptr<EncodedFrame> frame) {
tommidb23ea62017-03-03 07:21:18 -0800383 TRACE_EVENT0("webrtc", "FrameBuffer::InsertFrame");
philipel93e451b2016-10-06 12:25:13 +0200384 RTC_DCHECK(frame);
tommi0a735642017-03-14 06:23:57 -0700385
Markus Handell6deec382020-07-07 12:17:12 +0200386 MutexLock lock(&mutex_);
philipel29f730e2017-03-15 08:10:08 -0700387
philipel9aa9b8d2021-02-15 13:31:29 +0100388 int64_t last_continuous_frame_id = last_continuous_frame_.value_or(-1);
philipele0b2f152016-09-28 10:23:49 +0200389
philipel112adf92017-06-15 09:06:21 -0700390 if (!ValidReferences(*frame)) {
philipel9aa9b8d2021-02-15 13:31:29 +0100391 RTC_LOG(LS_WARNING) << "Frame " << frame->Id()
philipelf1091932021-02-11 15:25:08 +0100392 << " has invalid frame references, dropping frame.";
philipel9aa9b8d2021-02-15 13:31:29 +0100393 return last_continuous_frame_id;
philipel112adf92017-06-15 09:06:21 -0700394 }
395
Ilya Nikolaevskiy6551faf2019-01-10 15:16:47 +0100396 if (frames_.size() >= kMaxFramesBuffered) {
philipel9771c502018-03-02 11:06:27 +0100397 if (frame->is_keyframe()) {
philipel9aa9b8d2021-02-15 13:31:29 +0100398 RTC_LOG(LS_WARNING) << "Inserting keyframe " << frame->Id()
philipelf1091932021-02-11 15:25:08 +0100399 << " but buffer is full, clearing"
Jonas Olssonb2b20312020-01-14 12:11:31 +0100400 " buffer and inserting the frame.";
philipel9771c502018-03-02 11:06:27 +0100401 ClearFramesAndHistory();
402 } else {
philipel9aa9b8d2021-02-15 13:31:29 +0100403 RTC_LOG(LS_WARNING) << "Frame " << frame->Id()
philipelf1091932021-02-11 15:25:08 +0100404 << " could not be inserted due to the frame "
Jonas Olssonb2b20312020-01-14 12:11:31 +0100405 "buffer being full, dropping frame.";
philipel9aa9b8d2021-02-15 13:31:29 +0100406 return last_continuous_frame_id;
philipel9771c502018-03-02 11:06:27 +0100407 }
philipele0b2f152016-09-28 10:23:49 +0200408 }
409
Ilya Nikolaevskiy13717842019-01-14 13:24:22 +0100410 auto last_decoded_frame = decoded_frames_history_.GetLastDecodedFrameId();
411 auto last_decoded_frame_timestamp =
412 decoded_frames_history_.GetLastDecodedFrameTimestamp();
philipel9aa9b8d2021-02-15 13:31:29 +0100413 if (last_decoded_frame && frame->Id() <= *last_decoded_frame) {
Ilya Nikolaevskiy13717842019-01-14 13:24:22 +0100414 if (AheadOf(frame->Timestamp(), *last_decoded_frame_timestamp) &&
philipel3042c2d2017-08-18 04:55:02 -0700415 frame->is_keyframe()) {
philipel9aa9b8d2021-02-15 13:31:29 +0100416 // If this frame has a newer timestamp but an earlier frame id then we
417 // assume there has been a jump in the frame id due to some encoder
philipelfcc60062017-01-18 05:35:20 -0800418 // reconfiguration or some other reason. Even though this is not according
419 // to spec we can still continue to decode from this frame if it is a
420 // keyframe.
Mirko Bonadei675513b2017-11-09 11:09:25 +0100421 RTC_LOG(LS_WARNING)
philipel9aa9b8d2021-02-15 13:31:29 +0100422 << "A jump in frame id was detected, clearing buffer.";
philipelfcc60062017-01-18 05:35:20 -0800423 ClearFramesAndHistory();
philipel9aa9b8d2021-02-15 13:31:29 +0100424 last_continuous_frame_id = -1;
philipelfcc60062017-01-18 05:35:20 -0800425 } else {
philipel9aa9b8d2021-02-15 13:31:29 +0100426 RTC_LOG(LS_WARNING) << "Frame " << frame->Id() << " inserted after frame "
427 << *last_decoded_frame
philipelf1091932021-02-11 15:25:08 +0100428 << " was handed off for decoding, dropping frame.";
philipel9aa9b8d2021-02-15 13:31:29 +0100429 return last_continuous_frame_id;
philipelfcc60062017-01-18 05:35:20 -0800430 }
philipele0b2f152016-09-28 10:23:49 +0200431 }
432
philipel146a48b2017-04-20 04:04:38 -0700433 // Test if inserting this frame would cause the order of the frames to become
434 // ambiguous (covering more than half the interval of 2^16). This can happen
philipel9aa9b8d2021-02-15 13:31:29 +0100435 // when the frame id make large jumps mid stream.
436 if (!frames_.empty() && frame->Id() < frames_.begin()->first &&
437 frames_.rbegin()->first < frame->Id()) {
438 RTC_LOG(LS_WARNING) << "A jump in frame id was detected, clearing buffer.";
philipel146a48b2017-04-20 04:04:38 -0700439 ClearFramesAndHistory();
philipel9aa9b8d2021-02-15 13:31:29 +0100440 last_continuous_frame_id = -1;
philipel146a48b2017-04-20 04:04:38 -0700441 }
442
philipel9aa9b8d2021-02-15 13:31:29 +0100443 auto info = frames_.emplace(frame->Id(), FrameInfo()).first;
philipele0b2f152016-09-28 10:23:49 +0200444
philipel93e451b2016-10-06 12:25:13 +0200445 if (info->second.frame) {
philipel9aa9b8d2021-02-15 13:31:29 +0100446 return last_continuous_frame_id;
philipele0b2f152016-09-28 10:23:49 +0200447 }
448
philipel93e451b2016-10-06 12:25:13 +0200449 if (!UpdateFrameInfoWithIncomingFrame(*frame, info))
philipel9aa9b8d2021-02-15 13:31:29 +0100450 return last_continuous_frame_id;
Ruslan Burakov493a6502019-02-27 15:32:48 +0100451
Evan Shrubsoled6cdf802022-03-02 15:13:55 +0100452 // If ReceiveTime is negative then it is not a valid timestamp.
453 if (!frame->delayed_by_retransmission() && frame->ReceivedTime() >= 0)
454 timing_->IncomingTimestamp(frame->Timestamp(),
455 Timestamp::Millis(frame->ReceivedTime()));
philipel0a9f6de2018-02-28 11:29:47 +0100456
philipel0cb73262020-12-08 17:36:53 +0100457 // It can happen that a frame will be reported as fully received even if a
458 // lower spatial layer frame is missing.
459 if (stats_callback_ && frame->is_last_spatial_layer) {
Johannes Kronb88b44e2019-08-22 13:16:44 +0200460 stats_callback_->OnCompleteFrame(frame->is_keyframe(), frame->size(),
461 frame->contentType());
462 }
463
philipele0b2f152016-09-28 10:23:49 +0200464 info->second.frame = std::move(frame);
philipele0b2f152016-09-28 10:23:49 +0200465
466 if (info->second.num_missing_continuous == 0) {
467 info->second.continuous = true;
468 PropagateContinuity(info);
philipel9aa9b8d2021-02-15 13:31:29 +0100469 last_continuous_frame_id = *last_continuous_frame_;
philipele0b2f152016-09-28 10:23:49 +0200470
471 // Since we now have new continuous frames there might be a better frame
Sebastian Jansson11d0d7b2019-04-11 12:39:34 +0200472 // to return from NextFrame.
Sebastian Jansson11d0d7b2019-04-11 12:39:34 +0200473 if (callback_queue_) {
474 callback_queue_->PostTask([this] {
Markus Handell6deec382020-07-07 12:17:12 +0200475 MutexLock lock(&mutex_);
Sebastian Jansson11d0d7b2019-04-11 12:39:34 +0200476 if (!callback_task_.Running())
477 return;
478 RTC_CHECK(frame_handler_);
479 callback_task_.Stop();
480 StartWaitForNextFrameOnQueue();
481 });
482 }
philipele0b2f152016-09-28 10:23:49 +0200483 }
484
philipel9aa9b8d2021-02-15 13:31:29 +0100485 return last_continuous_frame_id;
philipelbe7a9e52016-05-19 12:19:35 +0200486}
487
philipele0b2f152016-09-28 10:23:49 +0200488void FrameBuffer::PropagateContinuity(FrameMap::iterator start) {
tommidb23ea62017-03-03 07:21:18 -0800489 TRACE_EVENT0("webrtc", "FrameBuffer::PropagateContinuity");
philipele0b2f152016-09-28 10:23:49 +0200490 RTC_DCHECK(start->second.continuous);
philipele0b2f152016-09-28 10:23:49 +0200491
492 std::queue<FrameMap::iterator> continuous_frames;
493 continuous_frames.push(start);
494
495 // A simple BFS to traverse continuous frames.
496 while (!continuous_frames.empty()) {
497 auto frame = continuous_frames.front();
498 continuous_frames.pop();
499
Ilya Nikolaevskiy6551faf2019-01-10 15:16:47 +0100500 if (!last_continuous_frame_ || *last_continuous_frame_ < frame->first) {
501 last_continuous_frame_ = frame->first;
502 }
philipele0b2f152016-09-28 10:23:49 +0200503
504 // Loop through all dependent frames, and if that frame no longer has
505 // any unfulfilled dependencies then that frame is continuous as well.
Elad Alon69321dd2019-01-10 15:02:54 +0100506 for (size_t d = 0; d < frame->second.dependent_frames.size(); ++d) {
philipele0b2f152016-09-28 10:23:49 +0200507 auto frame_ref = frames_.find(frame->second.dependent_frames[d]);
philipel112adf92017-06-15 09:06:21 -0700508 RTC_DCHECK(frame_ref != frames_.end());
philipele0b2f152016-09-28 10:23:49 +0200509
philipel112adf92017-06-15 09:06:21 -0700510 // TODO(philipel): Look into why we've seen this happen.
511 if (frame_ref != frames_.end()) {
512 --frame_ref->second.num_missing_continuous;
513 if (frame_ref->second.num_missing_continuous == 0) {
514 frame_ref->second.continuous = true;
515 continuous_frames.push(frame_ref);
516 }
philipele0b2f152016-09-28 10:23:49 +0200517 }
518 }
519 }
520}
521
522void FrameBuffer::PropagateDecodability(const FrameInfo& info) {
tommidb23ea62017-03-03 07:21:18 -0800523 TRACE_EVENT0("webrtc", "FrameBuffer::PropagateDecodability");
Elad Alon69321dd2019-01-10 15:02:54 +0100524 for (size_t d = 0; d < info.dependent_frames.size(); ++d) {
philipele0b2f152016-09-28 10:23:49 +0200525 auto ref_info = frames_.find(info.dependent_frames[d]);
philipel93e451b2016-10-06 12:25:13 +0200526 RTC_DCHECK(ref_info != frames_.end());
tommie95b78b2017-05-14 07:23:11 -0700527 // TODO(philipel): Look into why we've seen this happen.
528 if (ref_info != frames_.end()) {
529 RTC_DCHECK_GT(ref_info->second.num_missing_decodable, 0U);
530 --ref_info->second.num_missing_decodable;
531 }
philipele0b2f152016-09-28 10:23:49 +0200532 }
533}
534
philipele7c891f2018-02-22 14:35:06 +0100535bool FrameBuffer::UpdateFrameInfoWithIncomingFrame(const EncodedFrame& frame,
philipele0b2f152016-09-28 10:23:49 +0200536 FrameMap::iterator info) {
tommidb23ea62017-03-03 07:21:18 -0800537 TRACE_EVENT0("webrtc", "FrameBuffer::UpdateFrameInfoWithIncomingFrame");
Ilya Nikolaevskiy13717842019-01-14 13:24:22 +0100538 auto last_decoded_frame = decoded_frames_history_.GetLastDecodedFrameId();
539 RTC_DCHECK(!last_decoded_frame || *last_decoded_frame < info->first);
philipele0b2f152016-09-28 10:23:49 +0200540
Artem Titovdcd7fc72021-08-09 13:02:57 +0200541 // In this function we determine how many missing dependencies this `frame`
542 // has to become continuous/decodable. If a frame that this `frame` depend
philipel798b2822018-06-11 13:10:14 +0200543 // on has already been decoded then we can ignore that dependency since it has
544 // already been fulfilled.
545 //
Artem Titovdcd7fc72021-08-09 13:02:57 +0200546 // For all other frames we will register a backwards reference to this `frame`
547 // so that `num_missing_continuous` and `num_missing_decodable` can be
philipel798b2822018-06-11 13:10:14 +0200548 // decremented as frames become continuous/are decoded.
549 struct Dependency {
philipel9aa9b8d2021-02-15 13:31:29 +0100550 int64_t frame_id;
philipel798b2822018-06-11 13:10:14 +0200551 bool continuous;
552 };
553 std::vector<Dependency> not_yet_fulfilled_dependencies;
554
555 // Find all dependencies that have not yet been fulfilled.
philipele0b2f152016-09-28 10:23:49 +0200556 for (size_t i = 0; i < frame.num_references; ++i) {
Artem Titovdcd7fc72021-08-09 13:02:57 +0200557 // Does `frame` depend on a frame earlier than the last decoded one?
philipel9aa9b8d2021-02-15 13:31:29 +0100558 if (last_decoded_frame && frame.references[i] <= *last_decoded_frame) {
Artem Titovdcd7fc72021-08-09 13:02:57 +0200559 // Was that frame decoded? If not, this `frame` will never become
philipel798b2822018-06-11 13:10:14 +0200560 // decodable.
philipel9aa9b8d2021-02-15 13:31:29 +0100561 if (!decoded_frames_history_.WasDecoded(frame.references[i])) {
philipel65e1f942017-07-24 08:26:53 -0700562 int64_t now_ms = clock_->TimeInMilliseconds();
563 if (last_log_non_decoded_ms_ + kLogNonDecodedIntervalMs < now_ms) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100564 RTC_LOG(LS_WARNING)
philipel9aa9b8d2021-02-15 13:31:29 +0100565 << "Frame " << frame.Id()
philipelf1091932021-02-11 15:25:08 +0100566 << " depends on a non-decoded frame more previous than the last "
567 "decoded frame, dropping frame.";
philipel65e1f942017-07-24 08:26:53 -0700568 last_log_non_decoded_ms_ = now_ms;
569 }
philipele0b2f152016-09-28 10:23:49 +0200570 return false;
571 }
philipele0b2f152016-09-28 10:23:49 +0200572 } else {
philipel9aa9b8d2021-02-15 13:31:29 +0100573 auto ref_info = frames_.find(frame.references[i]);
philipel798b2822018-06-11 13:10:14 +0200574 bool ref_continuous =
575 ref_info != frames_.end() && ref_info->second.continuous;
philipel9aa9b8d2021-02-15 13:31:29 +0100576 not_yet_fulfilled_dependencies.push_back(
577 {frame.references[i], ref_continuous});
philipele0b2f152016-09-28 10:23:49 +0200578 }
philipelbe7a9e52016-05-19 12:19:35 +0200579 }
580
philipel798b2822018-06-11 13:10:14 +0200581 info->second.num_missing_continuous = not_yet_fulfilled_dependencies.size();
582 info->second.num_missing_decodable = not_yet_fulfilled_dependencies.size();
583
584 for (const Dependency& dep : not_yet_fulfilled_dependencies) {
585 if (dep.continuous)
586 --info->second.num_missing_continuous;
587
philipel9aa9b8d2021-02-15 13:31:29 +0100588 frames_[dep.frame_id].dependent_frames.push_back(frame.Id());
philipel798b2822018-06-11 13:10:14 +0200589 }
philipel93e451b2016-10-06 12:25:13 +0200590
philipelbe7a9e52016-05-19 12:19:35 +0200591 return true;
592}
593
philipelbe742702016-11-30 01:31:40 -0800594void FrameBuffer::UpdateJitterDelay() {
tommidb23ea62017-03-03 07:21:18 -0800595 TRACE_EVENT0("webrtc", "FrameBuffer::UpdateJitterDelay");
philipela45102f2017-02-22 05:30:39 -0800596 if (!stats_callback_)
597 return;
philipelbe742702016-11-30 01:31:40 -0800598
Evan Shrubsole92e89d72022-03-22 10:55:15 +0100599 auto timings = timing_->GetTimings();
600 if (timings.num_decoded_frames > 0) {
philipela45102f2017-02-22 05:30:39 -0800601 stats_callback_->OnFrameBufferTimingsUpdated(
Evan Shrubsole92e89d72022-03-22 10:55:15 +0100602 timings.max_decode_duration.ms(), timings.current_delay.ms(),
603 timings.target_delay.ms(), timings.jitter_buffer_delay.ms(),
604 timings.min_playout_delay.ms(), timings.render_delay.ms());
philipelbe742702016-11-30 01:31:40 -0800605 }
philipel266f0a42016-11-28 08:49:07 -0800606}
607
ilnik2edc6842017-07-06 03:06:50 -0700608void FrameBuffer::UpdateTimingFrameInfo() {
609 TRACE_EVENT0("webrtc", "FrameBuffer::UpdateTimingFrameInfo");
Danil Chapovalov0040b662018-06-18 10:48:16 +0200610 absl::optional<TimingFrameInfo> info = timing_->GetTimingFrameInfo();
philipel97187112018-03-23 10:43:21 +0100611 if (info && stats_callback_)
ilnik2edc6842017-07-06 03:06:50 -0700612 stats_callback_->OnTimingFrameInfoUpdated(*info);
613}
614
philipelfcc60062017-01-18 05:35:20 -0800615void FrameBuffer::ClearFramesAndHistory() {
ilnik2edc6842017-07-06 03:06:50 -0700616 TRACE_EVENT0("webrtc", "FrameBuffer::ClearFramesAndHistory");
Johannes Kron0c141c52019-08-26 15:04:43 +0200617 if (stats_callback_) {
philipel9aa9b8d2021-02-15 13:31:29 +0100618 unsigned int dropped_frames =
619 std::count_if(frames_.begin(), frames_.end(),
620 [](const std::pair<const int64_t, FrameInfo>& frame) {
621 return frame.second.frame != nullptr;
622 });
Johannes Kron0c141c52019-08-26 15:04:43 +0200623 if (dropped_frames > 0) {
624 stats_callback_->OnDroppedFrames(dropped_frames);
625 }
626 }
philipelfcc60062017-01-18 05:35:20 -0800627 frames_.clear();
Ilya Nikolaevskiy6551faf2019-01-10 15:16:47 +0100628 last_continuous_frame_.reset();
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100629 frames_to_decode_.clear();
Ilya Nikolaevskiy13717842019-01-14 13:24:22 +0100630 decoded_frames_history_.Clear();
philipelfcc60062017-01-18 05:35:20 -0800631}
632
Niels Möllerff2e2152019-09-27 10:29:30 +0200633// TODO(philipel): Avoid the concatenation of frames here, by replacing
634// NextFrame and GetNextFrame with methods returning multiple frames.
Evan Shrubsole0072c212021-11-10 11:41:14 +0100635std::unique_ptr<EncodedFrame> FrameBuffer::CombineAndDeleteFrames(
636 std::vector<std::unique_ptr<EncodedFrame>> frames) const {
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100637 RTC_DCHECK(!frames.empty());
Evan Shrubsolef83d4262021-12-15 15:05:15 +0100638 absl::InlinedVector<std::unique_ptr<EncodedFrame>, 4> inlined;
639 for (auto& frame : frames) {
640 inlined.push_back(std::move(frame));
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100641 }
Evan Shrubsolef83d4262021-12-15 15:05:15 +0100642 return webrtc::CombineAndDeleteFrames(std::move(inlined));
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100643}
644
Niels Möllerbe682d42018-03-27 08:31:45 +0200645FrameBuffer::FrameInfo::FrameInfo() = default;
646FrameBuffer::FrameInfo::FrameInfo(FrameInfo&&) = default;
647FrameBuffer::FrameInfo::~FrameInfo() = default;
648
philipelbe7a9e52016-05-19 12:19:35 +0200649} // namespace video_coding
650} // namespace webrtc