philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 11 | #include "modules/video_coding/frame_buffer2.h" |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 12 | |
| 13 | #include <algorithm> |
Yves Gerey | 3e70781 | 2018-11-28 16:47:49 +0100 | [diff] [blame] | 14 | #include <cstdlib> |
| 15 | #include <iterator> |
Evan Shrubsole | 0072c21 | 2021-11-10 11:41:14 +0100 | [diff] [blame] | 16 | #include <memory> |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 17 | #include <queue> |
Yves Gerey | 3e70781 | 2018-11-28 16:47:49 +0100 | [diff] [blame] | 18 | #include <utility> |
philipel | 798b282 | 2018-06-11 13:10:14 +0200 | [diff] [blame] | 19 | #include <vector> |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 20 | |
Evan Shrubsole | f83d426 | 2021-12-15 15:05:15 +0100 | [diff] [blame] | 21 | #include "absl/container/inlined_vector.h" |
Evan Shrubsole | 13e42a8 | 2022-03-07 13:21:51 +0100 | [diff] [blame] | 22 | #include "api/units/data_size.h" |
| 23 | #include "api/units/time_delta.h" |
Yves Gerey | 3e70781 | 2018-11-28 16:47:49 +0100 | [diff] [blame] | 24 | #include "api/video/encoded_image.h" |
| 25 | #include "api/video/video_timing.h" |
Evan Shrubsole | f83d426 | 2021-12-15 15:05:15 +0100 | [diff] [blame] | 26 | #include "modules/video_coding/frame_helpers.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 27 | #include "modules/video_coding/include/video_coding_defines.h" |
Rasmus Brandt | 10944e6 | 2022-05-25 10:12:42 +0200 | [diff] [blame] | 28 | #include "modules/video_coding/timing/jitter_estimator.h" |
Rasmus Brandt | c4d253c | 2022-05-25 12:03:35 +0200 | [diff] [blame^] | 29 | #include "modules/video_coding/timing/timing.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 30 | #include "rtc_base/checks.h" |
Yves Gerey | 3e70781 | 2018-11-28 16:47:49 +0100 | [diff] [blame] | 31 | #include "rtc_base/experiments/rtt_mult_experiment.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 32 | #include "rtc_base/logging.h" |
Yves Gerey | 3e70781 | 2018-11-28 16:47:49 +0100 | [diff] [blame] | 33 | #include "rtc_base/numerics/sequence_number_util.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 34 | #include "rtc_base/trace_event.h" |
| 35 | #include "system_wrappers/include/clock.h" |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 36 | |
| 37 | namespace webrtc { |
| 38 | namespace video_coding { |
| 39 | |
| 40 | namespace { |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 41 | // Max number of frames the buffer will hold. |
Ilya Nikolaevskiy | 1371784 | 2019-01-14 13:24:22 +0100 | [diff] [blame] | 42 | constexpr size_t kMaxFramesBuffered = 800; |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 43 | |
Johannes Kron | 23bfff3 | 2021-09-28 21:31:46 +0200 | [diff] [blame] | 44 | // Default value for the maximum decode queue size that is used when the |
| 45 | // low-latency renderer is used. |
| 46 | constexpr size_t kZeroPlayoutDelayDefaultMaxDecodeQueueSize = 8; |
| 47 | |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 48 | // Max number of decoded frame info that will be saved. |
Ilya Nikolaevskiy | 1371784 | 2019-01-14 13:24:22 +0100 | [diff] [blame] | 49 | constexpr int kMaxFramesHistory = 1 << 13; |
philipel | 65e1f94 | 2017-07-24 08:26:53 -0700 | [diff] [blame] | 50 | |
Ilya Nikolaevskiy | 8c4fe16 | 2018-02-27 15:49:47 +0100 | [diff] [blame] | 51 | // The time it's allowed for a frame to be late to its rendering prediction and |
| 52 | // still be rendered. |
Ilya Nikolaevskiy | 7eef007 | 2018-02-28 09:59:26 +0100 | [diff] [blame] | 53 | constexpr int kMaxAllowedFrameDelayMs = 5; |
Ilya Nikolaevskiy | 8c4fe16 | 2018-02-27 15:49:47 +0100 | [diff] [blame] | 54 | |
philipel | 65e1f94 | 2017-07-24 08:26:53 -0700 | [diff] [blame] | 55 | constexpr int64_t kLogNonDecodedIntervalMs = 5000; |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 56 | } // namespace |
| 57 | |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 58 | FrameBuffer::FrameBuffer(Clock* clock, |
philipel | a45102f | 2017-02-22 05:30:39 -0800 | [diff] [blame] | 59 | VCMTiming* timing, |
Jonas Oreland | e02f9ee | 2022-03-25 12:43:14 +0100 | [diff] [blame] | 60 | VCMReceiveStatisticsCallback* stats_callback, |
Jonas Oreland | e62c2f2 | 2022-03-29 11:04:48 +0200 | [diff] [blame] | 61 | const FieldTrialsView& field_trials) |
Ilya Nikolaevskiy | 1371784 | 2019-01-14 13:24:22 +0100 | [diff] [blame] | 62 | : decoded_frames_history_(kMaxFramesHistory), |
| 63 | clock_(clock), |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 64 | callback_queue_(nullptr), |
Jonas Oreland | e02f9ee | 2022-03-25 12:43:14 +0100 | [diff] [blame] | 65 | jitter_estimator_(clock, field_trials), |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 66 | timing_(timing), |
philipel | 29f730e | 2017-03-15 08:10:08 -0700 | [diff] [blame] | 67 | stopped_(false), |
philipel | a45102f | 2017-02-22 05:30:39 -0800 | [diff] [blame] | 68 | protection_mode_(kProtectionNack), |
philipel | 65e1f94 | 2017-07-24 08:26:53 -0700 | [diff] [blame] | 69 | stats_callback_(stats_callback), |
Elad Alon | e4b5023 | 2019-01-14 18:56:14 +0100 | [diff] [blame] | 70 | last_log_non_decoded_ms_(-kLogNonDecodedIntervalMs), |
Johannes Kron | 2ddc39e | 2021-08-10 16:56:12 +0200 | [diff] [blame] | 71 | rtt_mult_settings_(RttMultExperiment::GetRttMultValue()), |
Johannes Kron | 23bfff3 | 2021-09-28 21:31:46 +0200 | [diff] [blame] | 72 | zero_playout_delay_max_decode_queue_size_( |
| 73 | "max_decode_queue_size", |
| 74 | kZeroPlayoutDelayDefaultMaxDecodeQueueSize) { |
Johannes Kron | 2ddc39e | 2021-08-10 16:56:12 +0200 | [diff] [blame] | 75 | ParseFieldTrial({&zero_playout_delay_max_decode_queue_size_}, |
Jonas Oreland | e02f9ee | 2022-03-25 12:43:14 +0100 | [diff] [blame] | 76 | field_trials.Lookup("WebRTC-ZeroPlayoutDelay")); |
Tommi | 430951a | 2020-05-19 23:27:29 +0200 | [diff] [blame] | 77 | callback_checker_.Detach(); |
| 78 | } |
philipel | 266f0a4 | 2016-11-28 08:49:07 -0800 | [diff] [blame] | 79 | |
Tommi | 430951a | 2020-05-19 23:27:29 +0200 | [diff] [blame] | 80 | FrameBuffer::~FrameBuffer() { |
| 81 | RTC_DCHECK_RUN_ON(&construction_checker_); |
| 82 | } |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 83 | |
Evan Shrubsole | 3d29efd | 2021-12-07 14:11:45 +0100 | [diff] [blame] | 84 | void FrameBuffer::NextFrame(int64_t max_wait_time_ms, |
| 85 | bool keyframe_required, |
| 86 | rtc::TaskQueue* callback_queue, |
| 87 | NextFrameCallback handler) { |
Tommi | 430951a | 2020-05-19 23:27:29 +0200 | [diff] [blame] | 88 | RTC_DCHECK_RUN_ON(&callback_checker_); |
| 89 | RTC_DCHECK(callback_queue->IsCurrent()); |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 90 | TRACE_EVENT0("webrtc", "FrameBuffer::NextFrame"); |
| 91 | int64_t latest_return_time_ms = |
| 92 | clock_->TimeInMilliseconds() + max_wait_time_ms; |
Tommi | 430951a | 2020-05-19 23:27:29 +0200 | [diff] [blame] | 93 | |
Markus Handell | 6deec38 | 2020-07-07 12:17:12 +0200 | [diff] [blame] | 94 | MutexLock lock(&mutex_); |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 95 | if (stopped_) { |
| 96 | return; |
| 97 | } |
| 98 | latest_return_time_ms_ = latest_return_time_ms; |
| 99 | keyframe_required_ = keyframe_required; |
| 100 | frame_handler_ = handler; |
| 101 | callback_queue_ = callback_queue; |
| 102 | StartWaitForNextFrameOnQueue(); |
| 103 | } |
| 104 | |
| 105 | void FrameBuffer::StartWaitForNextFrameOnQueue() { |
| 106 | RTC_DCHECK(callback_queue_); |
| 107 | RTC_DCHECK(!callback_task_.Running()); |
Evan Shrubsole | d6cdf80 | 2022-03-02 15:13:55 +0100 | [diff] [blame] | 108 | int64_t wait_ms = FindNextFrame(clock_->CurrentTime()); |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 109 | callback_task_ = RepeatingTaskHandle::DelayedStart( |
Henrik Boström | 4c72f99 | 2022-01-25 08:15:50 +0100 | [diff] [blame] | 110 | callback_queue_->Get(), TimeDelta::Millis(wait_ms), |
| 111 | [this] { |
Tommi | 430951a | 2020-05-19 23:27:29 +0200 | [diff] [blame] | 112 | RTC_DCHECK_RUN_ON(&callback_checker_); |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 113 | // If this task has not been cancelled, we did not get any new frames |
| 114 | // while waiting. Continue with frame delivery. |
Markus Handell | 334b1fd | 2020-12-14 00:08:11 +0100 | [diff] [blame] | 115 | std::unique_ptr<EncodedFrame> frame; |
Evan Shrubsole | 3d29efd | 2021-12-07 14:11:45 +0100 | [diff] [blame] | 116 | NextFrameCallback frame_handler; |
Markus Handell | 334b1fd | 2020-12-14 00:08:11 +0100 | [diff] [blame] | 117 | { |
| 118 | MutexLock lock(&mutex_); |
| 119 | if (!frames_to_decode_.empty()) { |
| 120 | // We have frames, deliver! |
Evan Shrubsole | 0072c21 | 2021-11-10 11:41:14 +0100 | [diff] [blame] | 121 | frame = GetNextFrame(); |
Evan Shrubsole | d6cdf80 | 2022-03-02 15:13:55 +0100 | [diff] [blame] | 122 | timing_->SetLastDecodeScheduledTimestamp(clock_->CurrentTime()); |
Markus Handell | 334b1fd | 2020-12-14 00:08:11 +0100 | [diff] [blame] | 123 | } else if (clock_->TimeInMilliseconds() < latest_return_time_ms_) { |
| 124 | // If there's no frames to decode and there is still time left, it |
| 125 | // means that the frame buffer was cleared between creation and |
| 126 | // execution of this task. Continue waiting for the remaining time. |
Evan Shrubsole | d6cdf80 | 2022-03-02 15:13:55 +0100 | [diff] [blame] | 127 | int64_t wait_ms = FindNextFrame(clock_->CurrentTime()); |
Markus Handell | 334b1fd | 2020-12-14 00:08:11 +0100 | [diff] [blame] | 128 | return TimeDelta::Millis(wait_ms); |
| 129 | } |
| 130 | frame_handler = std::move(frame_handler_); |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 131 | CancelCallback(); |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 132 | } |
Markus Handell | 334b1fd | 2020-12-14 00:08:11 +0100 | [diff] [blame] | 133 | // Deliver frame, if any. Otherwise signal timeout. |
Evan Shrubsole | 3d29efd | 2021-12-07 14:11:45 +0100 | [diff] [blame] | 134 | frame_handler(std::move(frame)); |
Markus Handell | 334b1fd | 2020-12-14 00:08:11 +0100 | [diff] [blame] | 135 | return TimeDelta::Zero(); // Ignored. |
Henrik Boström | 4c72f99 | 2022-01-25 08:15:50 +0100 | [diff] [blame] | 136 | }, |
| 137 | TaskQueueBase::DelayPrecision::kHigh); |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 138 | } |
| 139 | |
Evan Shrubsole | d6cdf80 | 2022-03-02 15:13:55 +0100 | [diff] [blame] | 140 | int64_t FrameBuffer::FindNextFrame(Timestamp now) { |
| 141 | int64_t wait_ms = latest_return_time_ms_ - now.ms(); |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 142 | frames_to_decode_.clear(); |
| 143 | |
Artem Titov | dcd7fc7 | 2021-08-09 13:02:57 +0200 | [diff] [blame] | 144 | // `last_continuous_frame_` may be empty below, but nullopt is smaller |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 145 | // than everything else and loop will immediately terminate as expected. |
| 146 | for (auto frame_it = frames_.begin(); |
| 147 | frame_it != frames_.end() && frame_it->first <= last_continuous_frame_; |
| 148 | ++frame_it) { |
| 149 | if (!frame_it->second.continuous || |
| 150 | frame_it->second.num_missing_decodable > 0) { |
| 151 | continue; |
| 152 | } |
| 153 | |
| 154 | EncodedFrame* frame = frame_it->second.frame.get(); |
| 155 | |
| 156 | if (keyframe_required_ && !frame->is_keyframe()) |
| 157 | continue; |
| 158 | |
| 159 | auto last_decoded_frame_timestamp = |
| 160 | decoded_frames_history_.GetLastDecodedFrameTimestamp(); |
| 161 | |
| 162 | // TODO(https://bugs.webrtc.org/9974): consider removing this check |
| 163 | // as it may make a stream undecodable after a very long delay between |
| 164 | // frames. |
| 165 | if (last_decoded_frame_timestamp && |
| 166 | AheadOf(*last_decoded_frame_timestamp, frame->Timestamp())) { |
| 167 | continue; |
| 168 | } |
| 169 | |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 170 | // Gather all remaining frames for the same superframe. |
| 171 | std::vector<FrameMap::iterator> current_superframe; |
| 172 | current_superframe.push_back(frame_it); |
| 173 | bool last_layer_completed = frame_it->second.frame->is_last_spatial_layer; |
| 174 | FrameMap::iterator next_frame_it = frame_it; |
philipel | a65d785 | 2020-11-20 17:49:24 +0100 | [diff] [blame] | 175 | while (!last_layer_completed) { |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 176 | ++next_frame_it; |
philipel | a65d785 | 2020-11-20 17:49:24 +0100 | [diff] [blame] | 177 | |
| 178 | if (next_frame_it == frames_.end() || !next_frame_it->second.frame) { |
| 179 | break; |
| 180 | } |
| 181 | |
| 182 | if (next_frame_it->second.frame->Timestamp() != frame->Timestamp() || |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 183 | !next_frame_it->second.continuous) { |
| 184 | break; |
| 185 | } |
philipel | a65d785 | 2020-11-20 17:49:24 +0100 | [diff] [blame] | 186 | |
| 187 | if (next_frame_it->second.num_missing_decodable > 0) { |
philipel | cb327d9 | 2020-12-10 10:49:20 +0100 | [diff] [blame] | 188 | bool has_inter_layer_dependency = false; |
| 189 | for (size_t i = 0; i < EncodedFrame::kMaxFrameReferences && |
philipel | a65d785 | 2020-11-20 17:49:24 +0100 | [diff] [blame] | 190 | i < next_frame_it->second.frame->num_references; |
| 191 | ++i) { |
philipel | 9aa9b8d | 2021-02-15 13:31:29 +0100 | [diff] [blame] | 192 | if (next_frame_it->second.frame->references[i] >= frame_it->first) { |
philipel | a65d785 | 2020-11-20 17:49:24 +0100 | [diff] [blame] | 193 | has_inter_layer_dependency = true; |
philipel | cb327d9 | 2020-12-10 10:49:20 +0100 | [diff] [blame] | 194 | break; |
philipel | a65d785 | 2020-11-20 17:49:24 +0100 | [diff] [blame] | 195 | } |
| 196 | } |
| 197 | |
| 198 | // If the frame has an undecoded dependency that is not within the same |
philipel | cb327d9 | 2020-12-10 10:49:20 +0100 | [diff] [blame] | 199 | // temporal unit then this frame is not yet ready to be decoded. If it |
philipel | a65d785 | 2020-11-20 17:49:24 +0100 | [diff] [blame] | 200 | // is within the same temporal unit then the not yet decoded dependency |
| 201 | // is just a lower spatial frame, which is ok. |
| 202 | if (!has_inter_layer_dependency || |
| 203 | next_frame_it->second.num_missing_decodable > 1) { |
| 204 | break; |
| 205 | } |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 206 | } |
philipel | a65d785 | 2020-11-20 17:49:24 +0100 | [diff] [blame] | 207 | |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 208 | current_superframe.push_back(next_frame_it); |
| 209 | last_layer_completed = next_frame_it->second.frame->is_last_spatial_layer; |
| 210 | } |
| 211 | // Check if the current superframe is complete. |
| 212 | // TODO(bugs.webrtc.org/10064): consider returning all available to |
| 213 | // decode frames even if the superframe is not complete yet. |
| 214 | if (!last_layer_completed) { |
| 215 | continue; |
| 216 | } |
| 217 | |
| 218 | frames_to_decode_ = std::move(current_superframe); |
| 219 | |
Evan Shrubsole | d6cdf80 | 2022-03-02 15:13:55 +0100 | [diff] [blame] | 220 | absl::optional<Timestamp> render_time = frame->RenderTimestamp(); |
| 221 | if (!render_time) { |
| 222 | render_time = timing_->RenderTime(frame->Timestamp(), now); |
| 223 | frame->SetRenderTime(render_time->ms()); |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 224 | } |
Johannes Kron | 2ddc39e | 2021-08-10 16:56:12 +0200 | [diff] [blame] | 225 | bool too_many_frames_queued = |
| 226 | frames_.size() > zero_playout_delay_max_decode_queue_size_ ? true |
| 227 | : false; |
Evan Shrubsole | d6cdf80 | 2022-03-02 15:13:55 +0100 | [diff] [blame] | 228 | wait_ms = |
| 229 | timing_->MaxWaitingTime(*render_time, now, too_many_frames_queued).ms(); |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 230 | |
| 231 | // This will cause the frame buffer to prefer high framerate rather |
| 232 | // than high resolution in the case of the decoder not decoding fast |
| 233 | // enough and the stream has multiple spatial and temporal layers. |
| 234 | // For multiple temporal layers it may cause non-base layer frames to be |
| 235 | // skipped if they are late. |
| 236 | if (wait_ms < -kMaxAllowedFrameDelayMs) |
| 237 | continue; |
| 238 | |
| 239 | break; |
| 240 | } |
Evan Shrubsole | d6cdf80 | 2022-03-02 15:13:55 +0100 | [diff] [blame] | 241 | wait_ms = std::min<int64_t>(wait_ms, latest_return_time_ms_ - now.ms()); |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 242 | wait_ms = std::max<int64_t>(wait_ms, 0); |
| 243 | return wait_ms; |
| 244 | } |
| 245 | |
Evan Shrubsole | 0072c21 | 2021-11-10 11:41:14 +0100 | [diff] [blame] | 246 | std::unique_ptr<EncodedFrame> FrameBuffer::GetNextFrame() { |
Tommi | 430951a | 2020-05-19 23:27:29 +0200 | [diff] [blame] | 247 | RTC_DCHECK_RUN_ON(&callback_checker_); |
Evan Shrubsole | d6cdf80 | 2022-03-02 15:13:55 +0100 | [diff] [blame] | 248 | Timestamp now = clock_->CurrentTime(); |
Artem Titov | dcd7fc7 | 2021-08-09 13:02:57 +0200 | [diff] [blame] | 249 | // TODO(ilnik): remove `frames_out` use frames_to_decode_ directly. |
Evan Shrubsole | 0072c21 | 2021-11-10 11:41:14 +0100 | [diff] [blame] | 250 | std::vector<std::unique_ptr<EncodedFrame>> frames_out; |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 251 | |
| 252 | RTC_DCHECK(!frames_to_decode_.empty()); |
| 253 | bool superframe_delayed_by_retransmission = false; |
Evan Shrubsole | 13e42a8 | 2022-03-07 13:21:51 +0100 | [diff] [blame] | 254 | DataSize superframe_size = DataSize::Zero(); |
Evan Shrubsole | 5b9b9aa | 2021-11-01 10:28:31 +0100 | [diff] [blame] | 255 | const EncodedFrame& first_frame = *frames_to_decode_[0]->second.frame; |
Evan Shrubsole | d6cdf80 | 2022-03-02 15:13:55 +0100 | [diff] [blame] | 256 | absl::optional<Timestamp> render_time = first_frame.RenderTimestamp(); |
Evan Shrubsole | 5b9b9aa | 2021-11-01 10:28:31 +0100 | [diff] [blame] | 257 | int64_t receive_time_ms = first_frame.ReceivedTime(); |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 258 | // Gracefully handle bad RTP timestamps and render time issues. |
Evan Shrubsole | d6cdf80 | 2022-03-02 15:13:55 +0100 | [diff] [blame] | 259 | if (!render_time || |
| 260 | FrameHasBadRenderTiming(*render_time, now, timing_->TargetVideoDelay())) { |
Niels Möller | d9c2d94 | 2019-04-30 09:16:36 +0200 | [diff] [blame] | 261 | jitter_estimator_.Reset(); |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 262 | timing_->Reset(); |
Evan Shrubsole | d6cdf80 | 2022-03-02 15:13:55 +0100 | [diff] [blame] | 263 | render_time = timing_->RenderTime(first_frame.Timestamp(), now); |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 264 | } |
| 265 | |
| 266 | for (FrameMap::iterator& frame_it : frames_to_decode_) { |
| 267 | RTC_DCHECK(frame_it != frames_.end()); |
Evan Shrubsole | 0072c21 | 2021-11-10 11:41:14 +0100 | [diff] [blame] | 268 | std::unique_ptr<EncodedFrame> frame = std::move(frame_it->second.frame); |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 269 | |
Evan Shrubsole | d6cdf80 | 2022-03-02 15:13:55 +0100 | [diff] [blame] | 270 | frame->SetRenderTime(render_time->ms()); |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 271 | |
| 272 | superframe_delayed_by_retransmission |= frame->delayed_by_retransmission(); |
| 273 | receive_time_ms = std::max(receive_time_ms, frame->ReceivedTime()); |
Evan Shrubsole | 13e42a8 | 2022-03-07 13:21:51 +0100 | [diff] [blame] | 274 | superframe_size += DataSize::Bytes(frame->size()); |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 275 | |
| 276 | PropagateDecodability(frame_it->second); |
| 277 | decoded_frames_history_.InsertDecoded(frame_it->first, frame->Timestamp()); |
| 278 | |
| 279 | // Remove decoded frame and all undecoded frames before it. |
Johannes Kron | 0c141c5 | 2019-08-26 15:04:43 +0200 | [diff] [blame] | 280 | if (stats_callback_) { |
philipel | 9aa9b8d | 2021-02-15 13:31:29 +0100 | [diff] [blame] | 281 | unsigned int dropped_frames = |
| 282 | std::count_if(frames_.begin(), frame_it, |
| 283 | [](const std::pair<const int64_t, FrameInfo>& frame) { |
| 284 | return frame.second.frame != nullptr; |
| 285 | }); |
Johannes Kron | 0c141c5 | 2019-08-26 15:04:43 +0200 | [diff] [blame] | 286 | if (dropped_frames > 0) { |
| 287 | stats_callback_->OnDroppedFrames(dropped_frames); |
| 288 | } |
| 289 | } |
| 290 | |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 291 | frames_.erase(frames_.begin(), ++frame_it); |
| 292 | |
Evan Shrubsole | 0072c21 | 2021-11-10 11:41:14 +0100 | [diff] [blame] | 293 | frames_out.emplace_back(std::move(frame)); |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 294 | } |
| 295 | |
| 296 | if (!superframe_delayed_by_retransmission) { |
Evan Shrubsole | e9126c1 | 2022-03-07 14:50:51 +0100 | [diff] [blame] | 297 | auto frame_delay = inter_frame_delay_.CalculateDelay( |
| 298 | first_frame.Timestamp(), Timestamp::Millis(receive_time_ms)); |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 299 | |
Evan Shrubsole | e9126c1 | 2022-03-07 14:50:51 +0100 | [diff] [blame] | 300 | if (frame_delay) { |
| 301 | jitter_estimator_.UpdateEstimate(*frame_delay, superframe_size); |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 302 | } |
| 303 | |
| 304 | float rtt_mult = protection_mode_ == kProtectionNackFEC ? 0.0 : 1.0; |
Evan Shrubsole | 13e42a8 | 2022-03-07 13:21:51 +0100 | [diff] [blame] | 305 | absl::optional<TimeDelta> rtt_mult_add_cap_ms = absl::nullopt; |
“Michael | d3a4ebe | 2019-06-07 03:55:01 -0500 | [diff] [blame] | 306 | if (rtt_mult_settings_.has_value()) { |
| 307 | rtt_mult = rtt_mult_settings_->rtt_mult_setting; |
Evan Shrubsole | 13e42a8 | 2022-03-07 13:21:51 +0100 | [diff] [blame] | 308 | rtt_mult_add_cap_ms = |
| 309 | TimeDelta::Millis(rtt_mult_settings_->rtt_mult_add_cap_ms); |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 310 | } |
Evan Shrubsole | 13e42a8 | 2022-03-07 13:21:51 +0100 | [diff] [blame] | 311 | timing_->SetJitterDelay( |
| 312 | jitter_estimator_.GetJitterEstimate(rtt_mult, rtt_mult_add_cap_ms)); |
Evan Shrubsole | d6cdf80 | 2022-03-02 15:13:55 +0100 | [diff] [blame] | 313 | timing_->UpdateCurrentDelay(*render_time, now); |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 314 | } else { |
Evan Shrubsole | 9146d76 | 2021-10-13 11:19:48 +0200 | [diff] [blame] | 315 | if (RttMultExperiment::RttMultEnabled()) |
Niels Möller | d9c2d94 | 2019-04-30 09:16:36 +0200 | [diff] [blame] | 316 | jitter_estimator_.FrameNacked(); |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 317 | } |
| 318 | |
| 319 | UpdateJitterDelay(); |
| 320 | UpdateTimingFrameInfo(); |
| 321 | |
| 322 | if (frames_out.size() == 1) { |
Evan Shrubsole | 0072c21 | 2021-11-10 11:41:14 +0100 | [diff] [blame] | 323 | return std::move(frames_out[0]); |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 324 | } else { |
Evan Shrubsole | 0072c21 | 2021-11-10 11:41:14 +0100 | [diff] [blame] | 325 | return CombineAndDeleteFrames(std::move(frames_out)); |
Sebastian Jansson | 1c747f5 | 2019-04-04 13:01:39 +0200 | [diff] [blame] | 326 | } |
| 327 | } |
| 328 | |
philipel | 4f6cd6a | 2016-08-03 10:59:32 +0200 | [diff] [blame] | 329 | void FrameBuffer::SetProtectionMode(VCMVideoProtection mode) { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 330 | TRACE_EVENT0("webrtc", "FrameBuffer::SetProtectionMode"); |
Markus Handell | 6deec38 | 2020-07-07 12:17:12 +0200 | [diff] [blame] | 331 | MutexLock lock(&mutex_); |
Henrik Boström | c680c4a | 2019-04-03 10:27:36 +0000 | [diff] [blame] | 332 | protection_mode_ = mode; |
philipel | 4f6cd6a | 2016-08-03 10:59:32 +0200 | [diff] [blame] | 333 | } |
| 334 | |
philipel | 504c47d | 2016-06-30 17:33:02 +0200 | [diff] [blame] | 335 | void FrameBuffer::Stop() { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 336 | TRACE_EVENT0("webrtc", "FrameBuffer::Stop"); |
Markus Handell | 6deec38 | 2020-07-07 12:17:12 +0200 | [diff] [blame] | 337 | MutexLock lock(&mutex_); |
Tommi | 430951a | 2020-05-19 23:27:29 +0200 | [diff] [blame] | 338 | if (stopped_) |
| 339 | return; |
Henrik Boström | c680c4a | 2019-04-03 10:27:36 +0000 | [diff] [blame] | 340 | stopped_ = true; |
Tommi | 430951a | 2020-05-19 23:27:29 +0200 | [diff] [blame] | 341 | |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 342 | CancelCallback(); |
philipel | 504c47d | 2016-06-30 17:33:02 +0200 | [diff] [blame] | 343 | } |
| 344 | |
Ilya Nikolaevskiy | e6a2d94 | 2018-11-07 14:32:28 +0100 | [diff] [blame] | 345 | void FrameBuffer::Clear() { |
Markus Handell | 6deec38 | 2020-07-07 12:17:12 +0200 | [diff] [blame] | 346 | MutexLock lock(&mutex_); |
Henrik Boström | c680c4a | 2019-04-03 10:27:36 +0000 | [diff] [blame] | 347 | ClearFramesAndHistory(); |
Ilya Nikolaevskiy | e6a2d94 | 2018-11-07 14:32:28 +0100 | [diff] [blame] | 348 | } |
| 349 | |
Johannes Kron | 111e981 | 2020-10-26 13:54:40 +0100 | [diff] [blame] | 350 | int FrameBuffer::Size() { |
| 351 | MutexLock lock(&mutex_); |
| 352 | return frames_.size(); |
| 353 | } |
| 354 | |
philipel | e21be1d | 2017-09-25 06:37:12 -0700 | [diff] [blame] | 355 | void FrameBuffer::UpdateRtt(int64_t rtt_ms) { |
Markus Handell | 6deec38 | 2020-07-07 12:17:12 +0200 | [diff] [blame] | 356 | MutexLock lock(&mutex_); |
Evan Shrubsole | 13e42a8 | 2022-03-07 13:21:51 +0100 | [diff] [blame] | 357 | jitter_estimator_.UpdateRtt(TimeDelta::Millis(rtt_ms)); |
philipel | e21be1d | 2017-09-25 06:37:12 -0700 | [diff] [blame] | 358 | } |
| 359 | |
philipel | e7c891f | 2018-02-22 14:35:06 +0100 | [diff] [blame] | 360 | bool FrameBuffer::ValidReferences(const EncodedFrame& frame) const { |
philipel | 112adf9 | 2017-06-15 09:06:21 -0700 | [diff] [blame] | 361 | for (size_t i = 0; i < frame.num_references; ++i) { |
philipel | 9aa9b8d | 2021-02-15 13:31:29 +0100 | [diff] [blame] | 362 | if (frame.references[i] >= frame.Id()) |
philipel | 112adf9 | 2017-06-15 09:06:21 -0700 | [diff] [blame] | 363 | return false; |
philipel | 3b3c9c4 | 2017-09-11 09:38:36 -0700 | [diff] [blame] | 364 | |
philipel | 112adf9 | 2017-06-15 09:06:21 -0700 | [diff] [blame] | 365 | for (size_t j = i + 1; j < frame.num_references; ++j) { |
| 366 | if (frame.references[i] == frame.references[j]) |
| 367 | return false; |
| 368 | } |
| 369 | } |
| 370 | |
philipel | 112adf9 | 2017-06-15 09:06:21 -0700 | [diff] [blame] | 371 | return true; |
| 372 | } |
| 373 | |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 374 | void FrameBuffer::CancelCallback() { |
Tommi | 430951a | 2020-05-19 23:27:29 +0200 | [diff] [blame] | 375 | // Called from the callback queue or from within Stop(). |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 376 | frame_handler_ = {}; |
| 377 | callback_task_.Stop(); |
| 378 | callback_queue_ = nullptr; |
Tommi | 430951a | 2020-05-19 23:27:29 +0200 | [diff] [blame] | 379 | callback_checker_.Detach(); |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 380 | } |
| 381 | |
philipel | e7c891f | 2018-02-22 14:35:06 +0100 | [diff] [blame] | 382 | int64_t FrameBuffer::InsertFrame(std::unique_ptr<EncodedFrame> frame) { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 383 | TRACE_EVENT0("webrtc", "FrameBuffer::InsertFrame"); |
philipel | 93e451b | 2016-10-06 12:25:13 +0200 | [diff] [blame] | 384 | RTC_DCHECK(frame); |
tommi | 0a73564 | 2017-03-14 06:23:57 -0700 | [diff] [blame] | 385 | |
Markus Handell | 6deec38 | 2020-07-07 12:17:12 +0200 | [diff] [blame] | 386 | MutexLock lock(&mutex_); |
philipel | 29f730e | 2017-03-15 08:10:08 -0700 | [diff] [blame] | 387 | |
philipel | 9aa9b8d | 2021-02-15 13:31:29 +0100 | [diff] [blame] | 388 | int64_t last_continuous_frame_id = last_continuous_frame_.value_or(-1); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 389 | |
philipel | 112adf9 | 2017-06-15 09:06:21 -0700 | [diff] [blame] | 390 | if (!ValidReferences(*frame)) { |
philipel | 9aa9b8d | 2021-02-15 13:31:29 +0100 | [diff] [blame] | 391 | RTC_LOG(LS_WARNING) << "Frame " << frame->Id() |
philipel | f109193 | 2021-02-11 15:25:08 +0100 | [diff] [blame] | 392 | << " has invalid frame references, dropping frame."; |
philipel | 9aa9b8d | 2021-02-15 13:31:29 +0100 | [diff] [blame] | 393 | return last_continuous_frame_id; |
philipel | 112adf9 | 2017-06-15 09:06:21 -0700 | [diff] [blame] | 394 | } |
| 395 | |
Ilya Nikolaevskiy | 6551faf | 2019-01-10 15:16:47 +0100 | [diff] [blame] | 396 | if (frames_.size() >= kMaxFramesBuffered) { |
philipel | 9771c50 | 2018-03-02 11:06:27 +0100 | [diff] [blame] | 397 | if (frame->is_keyframe()) { |
philipel | 9aa9b8d | 2021-02-15 13:31:29 +0100 | [diff] [blame] | 398 | RTC_LOG(LS_WARNING) << "Inserting keyframe " << frame->Id() |
philipel | f109193 | 2021-02-11 15:25:08 +0100 | [diff] [blame] | 399 | << " but buffer is full, clearing" |
Jonas Olsson | b2b2031 | 2020-01-14 12:11:31 +0100 | [diff] [blame] | 400 | " buffer and inserting the frame."; |
philipel | 9771c50 | 2018-03-02 11:06:27 +0100 | [diff] [blame] | 401 | ClearFramesAndHistory(); |
| 402 | } else { |
philipel | 9aa9b8d | 2021-02-15 13:31:29 +0100 | [diff] [blame] | 403 | RTC_LOG(LS_WARNING) << "Frame " << frame->Id() |
philipel | f109193 | 2021-02-11 15:25:08 +0100 | [diff] [blame] | 404 | << " could not be inserted due to the frame " |
Jonas Olsson | b2b2031 | 2020-01-14 12:11:31 +0100 | [diff] [blame] | 405 | "buffer being full, dropping frame."; |
philipel | 9aa9b8d | 2021-02-15 13:31:29 +0100 | [diff] [blame] | 406 | return last_continuous_frame_id; |
philipel | 9771c50 | 2018-03-02 11:06:27 +0100 | [diff] [blame] | 407 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 408 | } |
| 409 | |
Ilya Nikolaevskiy | 1371784 | 2019-01-14 13:24:22 +0100 | [diff] [blame] | 410 | auto last_decoded_frame = decoded_frames_history_.GetLastDecodedFrameId(); |
| 411 | auto last_decoded_frame_timestamp = |
| 412 | decoded_frames_history_.GetLastDecodedFrameTimestamp(); |
philipel | 9aa9b8d | 2021-02-15 13:31:29 +0100 | [diff] [blame] | 413 | if (last_decoded_frame && frame->Id() <= *last_decoded_frame) { |
Ilya Nikolaevskiy | 1371784 | 2019-01-14 13:24:22 +0100 | [diff] [blame] | 414 | if (AheadOf(frame->Timestamp(), *last_decoded_frame_timestamp) && |
philipel | 3042c2d | 2017-08-18 04:55:02 -0700 | [diff] [blame] | 415 | frame->is_keyframe()) { |
philipel | 9aa9b8d | 2021-02-15 13:31:29 +0100 | [diff] [blame] | 416 | // If this frame has a newer timestamp but an earlier frame id then we |
| 417 | // assume there has been a jump in the frame id due to some encoder |
philipel | fcc6006 | 2017-01-18 05:35:20 -0800 | [diff] [blame] | 418 | // reconfiguration or some other reason. Even though this is not according |
| 419 | // to spec we can still continue to decode from this frame if it is a |
| 420 | // keyframe. |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 421 | RTC_LOG(LS_WARNING) |
philipel | 9aa9b8d | 2021-02-15 13:31:29 +0100 | [diff] [blame] | 422 | << "A jump in frame id was detected, clearing buffer."; |
philipel | fcc6006 | 2017-01-18 05:35:20 -0800 | [diff] [blame] | 423 | ClearFramesAndHistory(); |
philipel | 9aa9b8d | 2021-02-15 13:31:29 +0100 | [diff] [blame] | 424 | last_continuous_frame_id = -1; |
philipel | fcc6006 | 2017-01-18 05:35:20 -0800 | [diff] [blame] | 425 | } else { |
philipel | 9aa9b8d | 2021-02-15 13:31:29 +0100 | [diff] [blame] | 426 | RTC_LOG(LS_WARNING) << "Frame " << frame->Id() << " inserted after frame " |
| 427 | << *last_decoded_frame |
philipel | f109193 | 2021-02-11 15:25:08 +0100 | [diff] [blame] | 428 | << " was handed off for decoding, dropping frame."; |
philipel | 9aa9b8d | 2021-02-15 13:31:29 +0100 | [diff] [blame] | 429 | return last_continuous_frame_id; |
philipel | fcc6006 | 2017-01-18 05:35:20 -0800 | [diff] [blame] | 430 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 431 | } |
| 432 | |
philipel | 146a48b | 2017-04-20 04:04:38 -0700 | [diff] [blame] | 433 | // Test if inserting this frame would cause the order of the frames to become |
| 434 | // ambiguous (covering more than half the interval of 2^16). This can happen |
philipel | 9aa9b8d | 2021-02-15 13:31:29 +0100 | [diff] [blame] | 435 | // when the frame id make large jumps mid stream. |
| 436 | if (!frames_.empty() && frame->Id() < frames_.begin()->first && |
| 437 | frames_.rbegin()->first < frame->Id()) { |
| 438 | RTC_LOG(LS_WARNING) << "A jump in frame id was detected, clearing buffer."; |
philipel | 146a48b | 2017-04-20 04:04:38 -0700 | [diff] [blame] | 439 | ClearFramesAndHistory(); |
philipel | 9aa9b8d | 2021-02-15 13:31:29 +0100 | [diff] [blame] | 440 | last_continuous_frame_id = -1; |
philipel | 146a48b | 2017-04-20 04:04:38 -0700 | [diff] [blame] | 441 | } |
| 442 | |
philipel | 9aa9b8d | 2021-02-15 13:31:29 +0100 | [diff] [blame] | 443 | auto info = frames_.emplace(frame->Id(), FrameInfo()).first; |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 444 | |
philipel | 93e451b | 2016-10-06 12:25:13 +0200 | [diff] [blame] | 445 | if (info->second.frame) { |
philipel | 9aa9b8d | 2021-02-15 13:31:29 +0100 | [diff] [blame] | 446 | return last_continuous_frame_id; |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 447 | } |
| 448 | |
philipel | 93e451b | 2016-10-06 12:25:13 +0200 | [diff] [blame] | 449 | if (!UpdateFrameInfoWithIncomingFrame(*frame, info)) |
philipel | 9aa9b8d | 2021-02-15 13:31:29 +0100 | [diff] [blame] | 450 | return last_continuous_frame_id; |
Ruslan Burakov | 493a650 | 2019-02-27 15:32:48 +0100 | [diff] [blame] | 451 | |
Evan Shrubsole | d6cdf80 | 2022-03-02 15:13:55 +0100 | [diff] [blame] | 452 | // If ReceiveTime is negative then it is not a valid timestamp. |
| 453 | if (!frame->delayed_by_retransmission() && frame->ReceivedTime() >= 0) |
| 454 | timing_->IncomingTimestamp(frame->Timestamp(), |
| 455 | Timestamp::Millis(frame->ReceivedTime())); |
philipel | 0a9f6de | 2018-02-28 11:29:47 +0100 | [diff] [blame] | 456 | |
philipel | 0cb7326 | 2020-12-08 17:36:53 +0100 | [diff] [blame] | 457 | // It can happen that a frame will be reported as fully received even if a |
| 458 | // lower spatial layer frame is missing. |
| 459 | if (stats_callback_ && frame->is_last_spatial_layer) { |
Johannes Kron | b88b44e | 2019-08-22 13:16:44 +0200 | [diff] [blame] | 460 | stats_callback_->OnCompleteFrame(frame->is_keyframe(), frame->size(), |
| 461 | frame->contentType()); |
| 462 | } |
| 463 | |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 464 | info->second.frame = std::move(frame); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 465 | |
| 466 | if (info->second.num_missing_continuous == 0) { |
| 467 | info->second.continuous = true; |
| 468 | PropagateContinuity(info); |
philipel | 9aa9b8d | 2021-02-15 13:31:29 +0100 | [diff] [blame] | 469 | last_continuous_frame_id = *last_continuous_frame_; |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 470 | |
| 471 | // Since we now have new continuous frames there might be a better frame |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 472 | // to return from NextFrame. |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 473 | if (callback_queue_) { |
| 474 | callback_queue_->PostTask([this] { |
Markus Handell | 6deec38 | 2020-07-07 12:17:12 +0200 | [diff] [blame] | 475 | MutexLock lock(&mutex_); |
Sebastian Jansson | 11d0d7b | 2019-04-11 12:39:34 +0200 | [diff] [blame] | 476 | if (!callback_task_.Running()) |
| 477 | return; |
| 478 | RTC_CHECK(frame_handler_); |
| 479 | callback_task_.Stop(); |
| 480 | StartWaitForNextFrameOnQueue(); |
| 481 | }); |
| 482 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 483 | } |
| 484 | |
philipel | 9aa9b8d | 2021-02-15 13:31:29 +0100 | [diff] [blame] | 485 | return last_continuous_frame_id; |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 486 | } |
| 487 | |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 488 | void FrameBuffer::PropagateContinuity(FrameMap::iterator start) { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 489 | TRACE_EVENT0("webrtc", "FrameBuffer::PropagateContinuity"); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 490 | RTC_DCHECK(start->second.continuous); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 491 | |
| 492 | std::queue<FrameMap::iterator> continuous_frames; |
| 493 | continuous_frames.push(start); |
| 494 | |
| 495 | // A simple BFS to traverse continuous frames. |
| 496 | while (!continuous_frames.empty()) { |
| 497 | auto frame = continuous_frames.front(); |
| 498 | continuous_frames.pop(); |
| 499 | |
Ilya Nikolaevskiy | 6551faf | 2019-01-10 15:16:47 +0100 | [diff] [blame] | 500 | if (!last_continuous_frame_ || *last_continuous_frame_ < frame->first) { |
| 501 | last_continuous_frame_ = frame->first; |
| 502 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 503 | |
| 504 | // Loop through all dependent frames, and if that frame no longer has |
| 505 | // any unfulfilled dependencies then that frame is continuous as well. |
Elad Alon | 69321dd | 2019-01-10 15:02:54 +0100 | [diff] [blame] | 506 | for (size_t d = 0; d < frame->second.dependent_frames.size(); ++d) { |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 507 | auto frame_ref = frames_.find(frame->second.dependent_frames[d]); |
philipel | 112adf9 | 2017-06-15 09:06:21 -0700 | [diff] [blame] | 508 | RTC_DCHECK(frame_ref != frames_.end()); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 509 | |
philipel | 112adf9 | 2017-06-15 09:06:21 -0700 | [diff] [blame] | 510 | // TODO(philipel): Look into why we've seen this happen. |
| 511 | if (frame_ref != frames_.end()) { |
| 512 | --frame_ref->second.num_missing_continuous; |
| 513 | if (frame_ref->second.num_missing_continuous == 0) { |
| 514 | frame_ref->second.continuous = true; |
| 515 | continuous_frames.push(frame_ref); |
| 516 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 517 | } |
| 518 | } |
| 519 | } |
| 520 | } |
| 521 | |
| 522 | void FrameBuffer::PropagateDecodability(const FrameInfo& info) { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 523 | TRACE_EVENT0("webrtc", "FrameBuffer::PropagateDecodability"); |
Elad Alon | 69321dd | 2019-01-10 15:02:54 +0100 | [diff] [blame] | 524 | for (size_t d = 0; d < info.dependent_frames.size(); ++d) { |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 525 | auto ref_info = frames_.find(info.dependent_frames[d]); |
philipel | 93e451b | 2016-10-06 12:25:13 +0200 | [diff] [blame] | 526 | RTC_DCHECK(ref_info != frames_.end()); |
tommi | e95b78b | 2017-05-14 07:23:11 -0700 | [diff] [blame] | 527 | // TODO(philipel): Look into why we've seen this happen. |
| 528 | if (ref_info != frames_.end()) { |
| 529 | RTC_DCHECK_GT(ref_info->second.num_missing_decodable, 0U); |
| 530 | --ref_info->second.num_missing_decodable; |
| 531 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 532 | } |
| 533 | } |
| 534 | |
philipel | e7c891f | 2018-02-22 14:35:06 +0100 | [diff] [blame] | 535 | bool FrameBuffer::UpdateFrameInfoWithIncomingFrame(const EncodedFrame& frame, |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 536 | FrameMap::iterator info) { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 537 | TRACE_EVENT0("webrtc", "FrameBuffer::UpdateFrameInfoWithIncomingFrame"); |
Ilya Nikolaevskiy | 1371784 | 2019-01-14 13:24:22 +0100 | [diff] [blame] | 538 | auto last_decoded_frame = decoded_frames_history_.GetLastDecodedFrameId(); |
| 539 | RTC_DCHECK(!last_decoded_frame || *last_decoded_frame < info->first); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 540 | |
Artem Titov | dcd7fc7 | 2021-08-09 13:02:57 +0200 | [diff] [blame] | 541 | // In this function we determine how many missing dependencies this `frame` |
| 542 | // has to become continuous/decodable. If a frame that this `frame` depend |
philipel | 798b282 | 2018-06-11 13:10:14 +0200 | [diff] [blame] | 543 | // on has already been decoded then we can ignore that dependency since it has |
| 544 | // already been fulfilled. |
| 545 | // |
Artem Titov | dcd7fc7 | 2021-08-09 13:02:57 +0200 | [diff] [blame] | 546 | // For all other frames we will register a backwards reference to this `frame` |
| 547 | // so that `num_missing_continuous` and `num_missing_decodable` can be |
philipel | 798b282 | 2018-06-11 13:10:14 +0200 | [diff] [blame] | 548 | // decremented as frames become continuous/are decoded. |
| 549 | struct Dependency { |
philipel | 9aa9b8d | 2021-02-15 13:31:29 +0100 | [diff] [blame] | 550 | int64_t frame_id; |
philipel | 798b282 | 2018-06-11 13:10:14 +0200 | [diff] [blame] | 551 | bool continuous; |
| 552 | }; |
| 553 | std::vector<Dependency> not_yet_fulfilled_dependencies; |
| 554 | |
| 555 | // Find all dependencies that have not yet been fulfilled. |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 556 | for (size_t i = 0; i < frame.num_references; ++i) { |
Artem Titov | dcd7fc7 | 2021-08-09 13:02:57 +0200 | [diff] [blame] | 557 | // Does `frame` depend on a frame earlier than the last decoded one? |
philipel | 9aa9b8d | 2021-02-15 13:31:29 +0100 | [diff] [blame] | 558 | if (last_decoded_frame && frame.references[i] <= *last_decoded_frame) { |
Artem Titov | dcd7fc7 | 2021-08-09 13:02:57 +0200 | [diff] [blame] | 559 | // Was that frame decoded? If not, this `frame` will never become |
philipel | 798b282 | 2018-06-11 13:10:14 +0200 | [diff] [blame] | 560 | // decodable. |
philipel | 9aa9b8d | 2021-02-15 13:31:29 +0100 | [diff] [blame] | 561 | if (!decoded_frames_history_.WasDecoded(frame.references[i])) { |
philipel | 65e1f94 | 2017-07-24 08:26:53 -0700 | [diff] [blame] | 562 | int64_t now_ms = clock_->TimeInMilliseconds(); |
| 563 | if (last_log_non_decoded_ms_ + kLogNonDecodedIntervalMs < now_ms) { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 564 | RTC_LOG(LS_WARNING) |
philipel | 9aa9b8d | 2021-02-15 13:31:29 +0100 | [diff] [blame] | 565 | << "Frame " << frame.Id() |
philipel | f109193 | 2021-02-11 15:25:08 +0100 | [diff] [blame] | 566 | << " depends on a non-decoded frame more previous than the last " |
| 567 | "decoded frame, dropping frame."; |
philipel | 65e1f94 | 2017-07-24 08:26:53 -0700 | [diff] [blame] | 568 | last_log_non_decoded_ms_ = now_ms; |
| 569 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 570 | return false; |
| 571 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 572 | } else { |
philipel | 9aa9b8d | 2021-02-15 13:31:29 +0100 | [diff] [blame] | 573 | auto ref_info = frames_.find(frame.references[i]); |
philipel | 798b282 | 2018-06-11 13:10:14 +0200 | [diff] [blame] | 574 | bool ref_continuous = |
| 575 | ref_info != frames_.end() && ref_info->second.continuous; |
philipel | 9aa9b8d | 2021-02-15 13:31:29 +0100 | [diff] [blame] | 576 | not_yet_fulfilled_dependencies.push_back( |
| 577 | {frame.references[i], ref_continuous}); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 578 | } |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 579 | } |
| 580 | |
philipel | 798b282 | 2018-06-11 13:10:14 +0200 | [diff] [blame] | 581 | info->second.num_missing_continuous = not_yet_fulfilled_dependencies.size(); |
| 582 | info->second.num_missing_decodable = not_yet_fulfilled_dependencies.size(); |
| 583 | |
| 584 | for (const Dependency& dep : not_yet_fulfilled_dependencies) { |
| 585 | if (dep.continuous) |
| 586 | --info->second.num_missing_continuous; |
| 587 | |
philipel | 9aa9b8d | 2021-02-15 13:31:29 +0100 | [diff] [blame] | 588 | frames_[dep.frame_id].dependent_frames.push_back(frame.Id()); |
philipel | 798b282 | 2018-06-11 13:10:14 +0200 | [diff] [blame] | 589 | } |
philipel | 93e451b | 2016-10-06 12:25:13 +0200 | [diff] [blame] | 590 | |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 591 | return true; |
| 592 | } |
| 593 | |
philipel | be74270 | 2016-11-30 01:31:40 -0800 | [diff] [blame] | 594 | void FrameBuffer::UpdateJitterDelay() { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 595 | TRACE_EVENT0("webrtc", "FrameBuffer::UpdateJitterDelay"); |
philipel | a45102f | 2017-02-22 05:30:39 -0800 | [diff] [blame] | 596 | if (!stats_callback_) |
| 597 | return; |
philipel | be74270 | 2016-11-30 01:31:40 -0800 | [diff] [blame] | 598 | |
Evan Shrubsole | 92e89d7 | 2022-03-22 10:55:15 +0100 | [diff] [blame] | 599 | auto timings = timing_->GetTimings(); |
| 600 | if (timings.num_decoded_frames > 0) { |
philipel | a45102f | 2017-02-22 05:30:39 -0800 | [diff] [blame] | 601 | stats_callback_->OnFrameBufferTimingsUpdated( |
Evan Shrubsole | 92e89d7 | 2022-03-22 10:55:15 +0100 | [diff] [blame] | 602 | timings.max_decode_duration.ms(), timings.current_delay.ms(), |
| 603 | timings.target_delay.ms(), timings.jitter_buffer_delay.ms(), |
| 604 | timings.min_playout_delay.ms(), timings.render_delay.ms()); |
philipel | be74270 | 2016-11-30 01:31:40 -0800 | [diff] [blame] | 605 | } |
philipel | 266f0a4 | 2016-11-28 08:49:07 -0800 | [diff] [blame] | 606 | } |
| 607 | |
ilnik | 2edc684 | 2017-07-06 03:06:50 -0700 | [diff] [blame] | 608 | void FrameBuffer::UpdateTimingFrameInfo() { |
| 609 | TRACE_EVENT0("webrtc", "FrameBuffer::UpdateTimingFrameInfo"); |
Danil Chapovalov | 0040b66 | 2018-06-18 10:48:16 +0200 | [diff] [blame] | 610 | absl::optional<TimingFrameInfo> info = timing_->GetTimingFrameInfo(); |
philipel | 9718711 | 2018-03-23 10:43:21 +0100 | [diff] [blame] | 611 | if (info && stats_callback_) |
ilnik | 2edc684 | 2017-07-06 03:06:50 -0700 | [diff] [blame] | 612 | stats_callback_->OnTimingFrameInfoUpdated(*info); |
| 613 | } |
| 614 | |
philipel | fcc6006 | 2017-01-18 05:35:20 -0800 | [diff] [blame] | 615 | void FrameBuffer::ClearFramesAndHistory() { |
ilnik | 2edc684 | 2017-07-06 03:06:50 -0700 | [diff] [blame] | 616 | TRACE_EVENT0("webrtc", "FrameBuffer::ClearFramesAndHistory"); |
Johannes Kron | 0c141c5 | 2019-08-26 15:04:43 +0200 | [diff] [blame] | 617 | if (stats_callback_) { |
philipel | 9aa9b8d | 2021-02-15 13:31:29 +0100 | [diff] [blame] | 618 | unsigned int dropped_frames = |
| 619 | std::count_if(frames_.begin(), frames_.end(), |
| 620 | [](const std::pair<const int64_t, FrameInfo>& frame) { |
| 621 | return frame.second.frame != nullptr; |
| 622 | }); |
Johannes Kron | 0c141c5 | 2019-08-26 15:04:43 +0200 | [diff] [blame] | 623 | if (dropped_frames > 0) { |
| 624 | stats_callback_->OnDroppedFrames(dropped_frames); |
| 625 | } |
| 626 | } |
philipel | fcc6006 | 2017-01-18 05:35:20 -0800 | [diff] [blame] | 627 | frames_.clear(); |
Ilya Nikolaevskiy | 6551faf | 2019-01-10 15:16:47 +0100 | [diff] [blame] | 628 | last_continuous_frame_.reset(); |
Ilya Nikolaevskiy | 5546aef | 2018-12-04 15:54:52 +0100 | [diff] [blame] | 629 | frames_to_decode_.clear(); |
Ilya Nikolaevskiy | 1371784 | 2019-01-14 13:24:22 +0100 | [diff] [blame] | 630 | decoded_frames_history_.Clear(); |
philipel | fcc6006 | 2017-01-18 05:35:20 -0800 | [diff] [blame] | 631 | } |
| 632 | |
Niels Möller | ff2e215 | 2019-09-27 10:29:30 +0200 | [diff] [blame] | 633 | // TODO(philipel): Avoid the concatenation of frames here, by replacing |
| 634 | // NextFrame and GetNextFrame with methods returning multiple frames. |
Evan Shrubsole | 0072c21 | 2021-11-10 11:41:14 +0100 | [diff] [blame] | 635 | std::unique_ptr<EncodedFrame> FrameBuffer::CombineAndDeleteFrames( |
| 636 | std::vector<std::unique_ptr<EncodedFrame>> frames) const { |
Ilya Nikolaevskiy | 5546aef | 2018-12-04 15:54:52 +0100 | [diff] [blame] | 637 | RTC_DCHECK(!frames.empty()); |
Evan Shrubsole | f83d426 | 2021-12-15 15:05:15 +0100 | [diff] [blame] | 638 | absl::InlinedVector<std::unique_ptr<EncodedFrame>, 4> inlined; |
| 639 | for (auto& frame : frames) { |
| 640 | inlined.push_back(std::move(frame)); |
Ilya Nikolaevskiy | 5546aef | 2018-12-04 15:54:52 +0100 | [diff] [blame] | 641 | } |
Evan Shrubsole | f83d426 | 2021-12-15 15:05:15 +0100 | [diff] [blame] | 642 | return webrtc::CombineAndDeleteFrames(std::move(inlined)); |
Ilya Nikolaevskiy | 5546aef | 2018-12-04 15:54:52 +0100 | [diff] [blame] | 643 | } |
| 644 | |
Niels Möller | be682d4 | 2018-03-27 08:31:45 +0200 | [diff] [blame] | 645 | FrameBuffer::FrameInfo::FrameInfo() = default; |
| 646 | FrameBuffer::FrameInfo::FrameInfo(FrameInfo&&) = default; |
| 647 | FrameBuffer::FrameInfo::~FrameInfo() = default; |
| 648 | |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 649 | } // namespace video_coding |
| 650 | } // namespace webrtc |