philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 11 | #include "modules/video_coding/frame_buffer2.h" |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 12 | |
| 13 | #include <algorithm> |
Yves Gerey | 3e70781 | 2018-11-28 16:47:49 +0100 | [diff] [blame] | 14 | #include <cstdlib> |
| 15 | #include <iterator> |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 16 | #include <queue> |
Yves Gerey | 3e70781 | 2018-11-28 16:47:49 +0100 | [diff] [blame] | 17 | #include <utility> |
philipel | 798b282 | 2018-06-11 13:10:14 +0200 | [diff] [blame] | 18 | #include <vector> |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 19 | |
Yves Gerey | 3e70781 | 2018-11-28 16:47:49 +0100 | [diff] [blame] | 20 | #include "api/video/encoded_image.h" |
| 21 | #include "api/video/video_timing.h" |
| 22 | #include "common_types.h" // NOLINT(build/include) |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 23 | #include "modules/video_coding/include/video_coding_defines.h" |
| 24 | #include "modules/video_coding/jitter_estimator.h" |
| 25 | #include "modules/video_coding/timing.h" |
| 26 | #include "rtc_base/checks.h" |
Yves Gerey | 3e70781 | 2018-11-28 16:47:49 +0100 | [diff] [blame] | 27 | #include "rtc_base/experiments/rtt_mult_experiment.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 28 | #include "rtc_base/logging.h" |
Yves Gerey | 3e70781 | 2018-11-28 16:47:49 +0100 | [diff] [blame] | 29 | #include "rtc_base/numerics/sequence_number_util.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 30 | #include "rtc_base/trace_event.h" |
| 31 | #include "system_wrappers/include/clock.h" |
philipel | 707f278 | 2017-10-02 14:10:28 +0200 | [diff] [blame] | 32 | #include "system_wrappers/include/field_trial.h" |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 33 | |
| 34 | namespace webrtc { |
| 35 | namespace video_coding { |
| 36 | |
| 37 | namespace { |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 38 | // Max number of frames the buffer will hold. |
Ilya Nikolaevskiy | 1371784 | 2019-01-14 13:24:22 +0100 | [diff] [blame] | 39 | constexpr size_t kMaxFramesBuffered = 800; |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 40 | |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 41 | // Max number of decoded frame info that will be saved. |
Ilya Nikolaevskiy | 1371784 | 2019-01-14 13:24:22 +0100 | [diff] [blame] | 42 | constexpr int kMaxFramesHistory = 1 << 13; |
philipel | 65e1f94 | 2017-07-24 08:26:53 -0700 | [diff] [blame] | 43 | |
Ilya Nikolaevskiy | 8c4fe16 | 2018-02-27 15:49:47 +0100 | [diff] [blame] | 44 | // The time it's allowed for a frame to be late to its rendering prediction and |
| 45 | // still be rendered. |
Ilya Nikolaevskiy | 7eef007 | 2018-02-28 09:59:26 +0100 | [diff] [blame] | 46 | constexpr int kMaxAllowedFrameDelayMs = 5; |
Ilya Nikolaevskiy | 8c4fe16 | 2018-02-27 15:49:47 +0100 | [diff] [blame] | 47 | |
philipel | 65e1f94 | 2017-07-24 08:26:53 -0700 | [diff] [blame] | 48 | constexpr int64_t kLogNonDecodedIntervalMs = 5000; |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 49 | } // namespace |
| 50 | |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 51 | FrameBuffer::FrameBuffer(Clock* clock, |
| 52 | VCMJitterEstimator* jitter_estimator, |
philipel | a45102f | 2017-02-22 05:30:39 -0800 | [diff] [blame] | 53 | VCMTiming* timing, |
| 54 | VCMReceiveStatisticsCallback* stats_callback) |
Ilya Nikolaevskiy | 1371784 | 2019-01-14 13:24:22 +0100 | [diff] [blame] | 55 | : decoded_frames_history_(kMaxFramesHistory), |
| 56 | clock_(clock), |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 57 | jitter_estimator_(jitter_estimator), |
| 58 | timing_(timing), |
philipel | 4f6cd6a | 2016-08-03 10:59:32 +0200 | [diff] [blame] | 59 | inter_frame_delay_(clock_->TimeInMilliseconds()), |
philipel | 29f730e | 2017-03-15 08:10:08 -0700 | [diff] [blame] | 60 | stopped_(false), |
philipel | a45102f | 2017-02-22 05:30:39 -0800 | [diff] [blame] | 61 | protection_mode_(kProtectionNack), |
philipel | 65e1f94 | 2017-07-24 08:26:53 -0700 | [diff] [blame] | 62 | stats_callback_(stats_callback), |
Elad Alon | e4b5023 | 2019-01-14 18:56:14 +0100 | [diff] [blame] | 63 | last_log_non_decoded_ms_(-kLogNonDecodedIntervalMs), |
| 64 | add_rtt_to_playout_delay_( |
| 65 | webrtc::field_trial::IsEnabled("WebRTC-AddRttToPlayoutDelay")) {} |
philipel | 266f0a4 | 2016-11-28 08:49:07 -0800 | [diff] [blame] | 66 | |
philipel | a45102f | 2017-02-22 05:30:39 -0800 | [diff] [blame] | 67 | FrameBuffer::~FrameBuffer() {} |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 68 | |
philipel | 7556282 | 2016-09-05 10:57:41 +0200 | [diff] [blame] | 69 | FrameBuffer::ReturnReason FrameBuffer::NextFrame( |
| 70 | int64_t max_wait_time_ms, |
philipel | e7c891f | 2018-02-22 14:35:06 +0100 | [diff] [blame] | 71 | std::unique_ptr<EncodedFrame>* frame_out, |
philipel | 3042c2d | 2017-08-18 04:55:02 -0700 | [diff] [blame] | 72 | bool keyframe_required) { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 73 | TRACE_EVENT0("webrtc", "FrameBuffer::NextFrame"); |
philipel | 1c05625 | 2017-01-31 09:53:12 -0800 | [diff] [blame] | 74 | int64_t latest_return_time_ms = |
| 75 | clock_->TimeInMilliseconds() + max_wait_time_ms; |
philipel | 504c47d | 2016-06-30 17:33:02 +0200 | [diff] [blame] | 76 | int64_t wait_ms = max_wait_time_ms; |
philipel | 29f730e | 2017-03-15 08:10:08 -0700 | [diff] [blame] | 77 | int64_t now_ms = 0; |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 78 | |
| 79 | do { |
philipel | 29f730e | 2017-03-15 08:10:08 -0700 | [diff] [blame] | 80 | now_ms = clock_->TimeInMilliseconds(); |
philipel | 504c47d | 2016-06-30 17:33:02 +0200 | [diff] [blame] | 81 | { |
| 82 | rtc::CritScope lock(&crit_); |
tommi | 0a73564 | 2017-03-14 06:23:57 -0700 | [diff] [blame] | 83 | new_continuous_frame_event_.Reset(); |
philipel | 29f730e | 2017-03-15 08:10:08 -0700 | [diff] [blame] | 84 | if (stopped_) |
| 85 | return kStopped; |
| 86 | |
| 87 | wait_ms = max_wait_time_ms; |
| 88 | |
Ilya Nikolaevskiy | 5546aef | 2018-12-04 15:54:52 +0100 | [diff] [blame] | 89 | // Need to hold |crit_| in order to access frames_to_decode_. therefore we |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 90 | // set it here in the loop instead of outside the loop in order to not |
Ilya Nikolaevskiy | 5546aef | 2018-12-04 15:54:52 +0100 | [diff] [blame] | 91 | // acquire the lock unnecessarily. |
| 92 | frames_to_decode_.clear(); |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 93 | |
Ilya Nikolaevskiy | 6551faf | 2019-01-10 15:16:47 +0100 | [diff] [blame] | 94 | // |last_continuous_frame_| may be empty below, but nullopt is smaller |
| 95 | // than everything else and loop will immediately terminate as expected. |
| 96 | for (auto frame_it = frames_.begin(); |
| 97 | frame_it != frames_.end() && |
| 98 | frame_it->first <= last_continuous_frame_; |
philipel | 146a48b | 2017-04-20 04:04:38 -0700 | [diff] [blame] | 99 | ++frame_it) { |
philipel | 93e451b | 2016-10-06 12:25:13 +0200 | [diff] [blame] | 100 | if (!frame_it->second.continuous || |
| 101 | frame_it->second.num_missing_decodable > 0) { |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 102 | continue; |
philipel | 93e451b | 2016-10-06 12:25:13 +0200 | [diff] [blame] | 103 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 104 | |
philipel | e7c891f | 2018-02-22 14:35:06 +0100 | [diff] [blame] | 105 | EncodedFrame* frame = frame_it->second.frame.get(); |
philipel | 3042c2d | 2017-08-18 04:55:02 -0700 | [diff] [blame] | 106 | |
| 107 | if (keyframe_required && !frame->is_keyframe()) |
| 108 | continue; |
| 109 | |
Ilya Nikolaevskiy | 1371784 | 2019-01-14 13:24:22 +0100 | [diff] [blame] | 110 | auto last_decoded_frame_timestamp = |
| 111 | decoded_frames_history_.GetLastDecodedFrameTimestamp(); |
| 112 | |
Ilya Nikolaevskiy | e6a2d94 | 2018-11-07 14:32:28 +0100 | [diff] [blame] | 113 | // TODO(https://bugs.webrtc.org/9974): consider removing this check |
| 114 | // as it may make a stream undecodable after a very long delay between |
| 115 | // frames. |
Ilya Nikolaevskiy | 1371784 | 2019-01-14 13:24:22 +0100 | [diff] [blame] | 116 | if (last_decoded_frame_timestamp && |
| 117 | AheadOf(*last_decoded_frame_timestamp, frame->Timestamp())) { |
philipel | 6d21650 | 2018-10-22 14:36:45 +0200 | [diff] [blame] | 118 | continue; |
| 119 | } |
| 120 | |
Ilya Nikolaevskiy | 5546aef | 2018-12-04 15:54:52 +0100 | [diff] [blame] | 121 | // Only ever return all parts of a superframe. Therefore skip this |
| 122 | // frame if it's not a beginning of a superframe. |
| 123 | if (frame->inter_layer_predicted) { |
| 124 | continue; |
| 125 | } |
| 126 | |
| 127 | // Gather all remaining frames for the same superframe. |
| 128 | std::vector<FrameMap::iterator> current_superframe; |
| 129 | current_superframe.push_back(frame_it); |
| 130 | bool last_layer_completed = |
| 131 | frame_it->second.frame->is_last_spatial_layer; |
| 132 | FrameMap::iterator next_frame_it = frame_it; |
| 133 | while (true) { |
| 134 | ++next_frame_it; |
| 135 | if (next_frame_it == frames_.end() || |
| 136 | next_frame_it->first.picture_id != frame->id.picture_id || |
| 137 | !next_frame_it->second.continuous) { |
| 138 | break; |
| 139 | } |
| 140 | // Check if the next frame has some undecoded references other than |
| 141 | // the previous frame in the same superframe. |
| 142 | size_t num_allowed_undecoded_refs = |
| 143 | (next_frame_it->second.frame->inter_layer_predicted) ? 1 : 0; |
| 144 | if (next_frame_it->second.num_missing_decodable > |
| 145 | num_allowed_undecoded_refs) { |
| 146 | break; |
| 147 | } |
| 148 | // All frames in the superframe should have the same timestamp. |
| 149 | if (frame->Timestamp() != next_frame_it->second.frame->Timestamp()) { |
| 150 | RTC_LOG(LS_WARNING) |
| 151 | << "Frames in a single superframe have different" |
| 152 | " timestamps. Skipping undecodable superframe."; |
| 153 | break; |
| 154 | } |
| 155 | current_superframe.push_back(next_frame_it); |
| 156 | last_layer_completed = |
| 157 | next_frame_it->second.frame->is_last_spatial_layer; |
| 158 | } |
| 159 | // Check if the current superframe is complete. |
| 160 | // TODO(bugs.webrtc.org/10064): consider returning all available to |
| 161 | // decode frames even if the superframe is not complete yet. |
| 162 | if (!last_layer_completed) { |
| 163 | continue; |
| 164 | } |
| 165 | |
| 166 | frames_to_decode_ = std::move(current_superframe); |
| 167 | |
philipel | 6d21650 | 2018-10-22 14:36:45 +0200 | [diff] [blame] | 168 | if (frame->RenderTime() == -1) { |
Niels Möller | 2377588 | 2018-08-16 10:24:12 +0200 | [diff] [blame] | 169 | frame->SetRenderTime( |
| 170 | timing_->RenderTimeMs(frame->Timestamp(), now_ms)); |
philipel | 6d21650 | 2018-10-22 14:36:45 +0200 | [diff] [blame] | 171 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 172 | wait_ms = timing_->MaxWaitingTime(frame->RenderTime(), now_ms); |
| 173 | |
| 174 | // This will cause the frame buffer to prefer high framerate rather |
| 175 | // than high resolution in the case of the decoder not decoding fast |
| 176 | // enough and the stream has multiple spatial and temporal layers. |
Ilya Nikolaevskiy | 8c4fe16 | 2018-02-27 15:49:47 +0100 | [diff] [blame] | 177 | // For multiple temporal layers it may cause non-base layer frames to be |
| 178 | // skipped if they are late. |
Ilya Nikolaevskiy | 7eef007 | 2018-02-28 09:59:26 +0100 | [diff] [blame] | 179 | if (wait_ms < -kMaxAllowedFrameDelayMs) |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 180 | continue; |
| 181 | |
| 182 | break; |
| 183 | } |
| 184 | } // rtc::Critscope lock(&crit_); |
| 185 | |
philipel | 1c05625 | 2017-01-31 09:53:12 -0800 | [diff] [blame] | 186 | wait_ms = std::min<int64_t>(wait_ms, latest_return_time_ms - now_ms); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 187 | wait_ms = std::max<int64_t>(wait_ms, 0); |
tommi | 0a73564 | 2017-03-14 06:23:57 -0700 | [diff] [blame] | 188 | } while (new_continuous_frame_event_.Wait(wait_ms)); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 189 | |
philipel | 29f730e | 2017-03-15 08:10:08 -0700 | [diff] [blame] | 190 | { |
| 191 | rtc::CritScope lock(&crit_); |
| 192 | now_ms = clock_->TimeInMilliseconds(); |
Ilya Nikolaevskiy | 5546aef | 2018-12-04 15:54:52 +0100 | [diff] [blame] | 193 | std::vector<EncodedFrame*> frames_out; |
Ilya Nikolaevskiy | 1371784 | 2019-01-14 13:24:22 +0100 | [diff] [blame] | 194 | for (FrameMap::iterator& frame_it : frames_to_decode_) { |
Ilya Nikolaevskiy | 5546aef | 2018-12-04 15:54:52 +0100 | [diff] [blame] | 195 | RTC_DCHECK(frame_it != frames_.end()); |
| 196 | EncodedFrame* frame = frame_it->second.frame.release(); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 197 | |
philipel | 29f730e | 2017-03-15 08:10:08 -0700 | [diff] [blame] | 198 | if (!frame->delayed_by_retransmission()) { |
| 199 | int64_t frame_delay; |
philipel | e075430 | 2017-01-25 08:56:23 -0800 | [diff] [blame] | 200 | |
Niels Möller | 2377588 | 2018-08-16 10:24:12 +0200 | [diff] [blame] | 201 | if (inter_frame_delay_.CalculateDelay(frame->Timestamp(), &frame_delay, |
philipel | 29f730e | 2017-03-15 08:10:08 -0700 | [diff] [blame] | 202 | frame->ReceivedTime())) { |
| 203 | jitter_estimator_->UpdateEstimate(frame_delay, frame->size()); |
| 204 | } |
| 205 | |
| 206 | float rtt_mult = protection_mode_ == kProtectionNackFEC ? 0.0 : 1.0; |
“Michael | f9fc171 | 2018-08-27 10:08:58 -0500 | [diff] [blame] | 207 | if (RttMultExperiment::RttMultEnabled()) { |
| 208 | rtt_mult = RttMultExperiment::GetRttMultValue(); |
| 209 | } |
philipel | 29f730e | 2017-03-15 08:10:08 -0700 | [diff] [blame] | 210 | timing_->SetJitterDelay(jitter_estimator_->GetJitterEstimate(rtt_mult)); |
| 211 | timing_->UpdateCurrentDelay(frame->RenderTime(), now_ms); |
philipel | e21be1d | 2017-09-25 06:37:12 -0700 | [diff] [blame] | 212 | } else { |
Elad Alon | e4b5023 | 2019-01-14 18:56:14 +0100 | [diff] [blame] | 213 | if (RttMultExperiment::RttMultEnabled() || add_rtt_to_playout_delay_) |
philipel | 707f278 | 2017-10-02 14:10:28 +0200 | [diff] [blame] | 214 | jitter_estimator_->FrameNacked(); |
philipel | e075430 | 2017-01-25 08:56:23 -0800 | [diff] [blame] | 215 | } |
| 216 | |
stefan | 95e9754 | 2017-05-23 09:52:18 -0700 | [diff] [blame] | 217 | // Gracefully handle bad RTP timestamps and render time issues. |
| 218 | if (HasBadRenderTiming(*frame, now_ms)) { |
| 219 | jitter_estimator_->Reset(); |
| 220 | timing_->Reset(); |
Niels Möller | 2377588 | 2018-08-16 10:24:12 +0200 | [diff] [blame] | 221 | frame->SetRenderTime(timing_->RenderTimeMs(frame->Timestamp(), now_ms)); |
stefan | 95e9754 | 2017-05-23 09:52:18 -0700 | [diff] [blame] | 222 | } |
| 223 | |
philipel | 29f730e | 2017-03-15 08:10:08 -0700 | [diff] [blame] | 224 | UpdateJitterDelay(); |
ilnik | 2edc684 | 2017-07-06 03:06:50 -0700 | [diff] [blame] | 225 | UpdateTimingFrameInfo(); |
Ilya Nikolaevskiy | 5546aef | 2018-12-04 15:54:52 +0100 | [diff] [blame] | 226 | PropagateDecodability(frame_it->second); |
brandtr | 9078d8c | 2017-04-27 07:07:27 -0700 | [diff] [blame] | 227 | |
Ilya Nikolaevskiy | 1371784 | 2019-01-14 13:24:22 +0100 | [diff] [blame] | 228 | decoded_frames_history_.InsertDecoded(frame_it->first, |
| 229 | frame->Timestamp()); |
| 230 | |
| 231 | // Remove decoded frame and all undecoded frames before it. |
| 232 | frames_.erase(frames_.begin(), ++frame_it); |
| 233 | |
Ilya Nikolaevskiy | 5546aef | 2018-12-04 15:54:52 +0100 | [diff] [blame] | 234 | frames_out.push_back(frame); |
| 235 | } |
| 236 | |
| 237 | if (!frames_out.empty()) { |
| 238 | if (frames_out.size() == 1) { |
| 239 | frame_out->reset(frames_out[0]); |
| 240 | } else { |
| 241 | frame_out->reset(CombineAndDeleteFrames(frames_out)); |
| 242 | } |
philipel | 29f730e | 2017-03-15 08:10:08 -0700 | [diff] [blame] | 243 | return kFrameFound; |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 244 | } |
Ilya Nikolaevskiy | 5546aef | 2018-12-04 15:54:52 +0100 | [diff] [blame] | 245 | } // rtc::Critscope lock(&crit_) |
tommi | 0a73564 | 2017-03-14 06:23:57 -0700 | [diff] [blame] | 246 | |
| 247 | if (latest_return_time_ms - now_ms > 0) { |
philipel | 1c05625 | 2017-01-31 09:53:12 -0800 | [diff] [blame] | 248 | // If |next_frame_it_ == frames_.end()| and there is still time left, it |
| 249 | // means that the frame buffer was cleared as the thread in this function |
| 250 | // was waiting to acquire |crit_| in order to return. Wait for the |
| 251 | // remaining time and then return. |
| 252 | return NextFrame(latest_return_time_ms - now_ms, frame_out); |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 253 | } |
tommi | 0a73564 | 2017-03-14 06:23:57 -0700 | [diff] [blame] | 254 | return kTimeout; |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 255 | } |
| 256 | |
philipel | e7c891f | 2018-02-22 14:35:06 +0100 | [diff] [blame] | 257 | bool FrameBuffer::HasBadRenderTiming(const EncodedFrame& frame, |
| 258 | int64_t now_ms) { |
stefan | 95e9754 | 2017-05-23 09:52:18 -0700 | [diff] [blame] | 259 | // Assume that render timing errors are due to changes in the video stream. |
| 260 | int64_t render_time_ms = frame.RenderTimeMs(); |
Stefan Holmer | 812ceaf | 2018-05-15 13:00:10 +0200 | [diff] [blame] | 261 | // Zero render time means render immediately. |
| 262 | if (render_time_ms == 0) { |
| 263 | return false; |
| 264 | } |
stefan | 95e9754 | 2017-05-23 09:52:18 -0700 | [diff] [blame] | 265 | if (render_time_ms < 0) { |
| 266 | return true; |
| 267 | } |
Stefan Holmer | 812ceaf | 2018-05-15 13:00:10 +0200 | [diff] [blame] | 268 | const int64_t kMaxVideoDelayMs = 10000; |
stefan | 95e9754 | 2017-05-23 09:52:18 -0700 | [diff] [blame] | 269 | if (std::abs(render_time_ms - now_ms) > kMaxVideoDelayMs) { |
| 270 | int frame_delay = static_cast<int>(std::abs(render_time_ms - now_ms)); |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 271 | RTC_LOG(LS_WARNING) |
| 272 | << "A frame about to be decoded is out of the configured " |
| 273 | << "delay bounds (" << frame_delay << " > " << kMaxVideoDelayMs |
| 274 | << "). Resetting the video jitter buffer."; |
stefan | 95e9754 | 2017-05-23 09:52:18 -0700 | [diff] [blame] | 275 | return true; |
| 276 | } |
| 277 | if (static_cast<int>(timing_->TargetVideoDelay()) > kMaxVideoDelayMs) { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 278 | RTC_LOG(LS_WARNING) << "The video target delay has grown larger than " |
| 279 | << kMaxVideoDelayMs << " ms."; |
stefan | 95e9754 | 2017-05-23 09:52:18 -0700 | [diff] [blame] | 280 | return true; |
| 281 | } |
| 282 | return false; |
| 283 | } |
| 284 | |
philipel | 4f6cd6a | 2016-08-03 10:59:32 +0200 | [diff] [blame] | 285 | void FrameBuffer::SetProtectionMode(VCMVideoProtection mode) { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 286 | TRACE_EVENT0("webrtc", "FrameBuffer::SetProtectionMode"); |
philipel | 4f6cd6a | 2016-08-03 10:59:32 +0200 | [diff] [blame] | 287 | rtc::CritScope lock(&crit_); |
| 288 | protection_mode_ = mode; |
| 289 | } |
| 290 | |
philipel | 504c47d | 2016-06-30 17:33:02 +0200 | [diff] [blame] | 291 | void FrameBuffer::Start() { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 292 | TRACE_EVENT0("webrtc", "FrameBuffer::Start"); |
philipel | 29f730e | 2017-03-15 08:10:08 -0700 | [diff] [blame] | 293 | rtc::CritScope lock(&crit_); |
| 294 | stopped_ = false; |
philipel | 504c47d | 2016-06-30 17:33:02 +0200 | [diff] [blame] | 295 | } |
| 296 | |
| 297 | void FrameBuffer::Stop() { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 298 | TRACE_EVENT0("webrtc", "FrameBuffer::Stop"); |
philipel | 29f730e | 2017-03-15 08:10:08 -0700 | [diff] [blame] | 299 | rtc::CritScope lock(&crit_); |
| 300 | stopped_ = true; |
tommi | 0a73564 | 2017-03-14 06:23:57 -0700 | [diff] [blame] | 301 | new_continuous_frame_event_.Set(); |
philipel | 504c47d | 2016-06-30 17:33:02 +0200 | [diff] [blame] | 302 | } |
| 303 | |
Ilya Nikolaevskiy | e6a2d94 | 2018-11-07 14:32:28 +0100 | [diff] [blame] | 304 | void FrameBuffer::Clear() { |
| 305 | rtc::CritScope lock(&crit_); |
| 306 | ClearFramesAndHistory(); |
| 307 | } |
| 308 | |
philipel | e21be1d | 2017-09-25 06:37:12 -0700 | [diff] [blame] | 309 | void FrameBuffer::UpdateRtt(int64_t rtt_ms) { |
| 310 | rtc::CritScope lock(&crit_); |
| 311 | jitter_estimator_->UpdateRtt(rtt_ms); |
| 312 | } |
| 313 | |
philipel | e7c891f | 2018-02-22 14:35:06 +0100 | [diff] [blame] | 314 | bool FrameBuffer::ValidReferences(const EncodedFrame& frame) const { |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 315 | if (frame.id.picture_id < 0) |
philipel | 3b3c9c4 | 2017-09-11 09:38:36 -0700 | [diff] [blame] | 316 | return false; |
| 317 | |
philipel | 112adf9 | 2017-06-15 09:06:21 -0700 | [diff] [blame] | 318 | for (size_t i = 0; i < frame.num_references; ++i) { |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 319 | if (frame.references[i] < 0 || frame.references[i] >= frame.id.picture_id) |
philipel | 112adf9 | 2017-06-15 09:06:21 -0700 | [diff] [blame] | 320 | return false; |
philipel | 3b3c9c4 | 2017-09-11 09:38:36 -0700 | [diff] [blame] | 321 | |
philipel | 112adf9 | 2017-06-15 09:06:21 -0700 | [diff] [blame] | 322 | for (size_t j = i + 1; j < frame.num_references; ++j) { |
| 323 | if (frame.references[i] == frame.references[j]) |
| 324 | return false; |
| 325 | } |
| 326 | } |
| 327 | |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 328 | if (frame.inter_layer_predicted && frame.id.spatial_layer == 0) |
philipel | 112adf9 | 2017-06-15 09:06:21 -0700 | [diff] [blame] | 329 | return false; |
| 330 | |
| 331 | return true; |
| 332 | } |
| 333 | |
philipel | e7c891f | 2018-02-22 14:35:06 +0100 | [diff] [blame] | 334 | void FrameBuffer::UpdatePlayoutDelays(const EncodedFrame& frame) { |
gnish | b2a318b | 2017-05-10 09:21:33 -0700 | [diff] [blame] | 335 | TRACE_EVENT0("webrtc", "FrameBuffer::UpdatePlayoutDelays"); |
| 336 | PlayoutDelay playout_delay = frame.EncodedImage().playout_delay_; |
| 337 | if (playout_delay.min_ms >= 0) |
| 338 | timing_->set_min_playout_delay(playout_delay.min_ms); |
| 339 | |
| 340 | if (playout_delay.max_ms >= 0) |
| 341 | timing_->set_max_playout_delay(playout_delay.max_ms); |
philipel | 0a9f6de | 2018-02-28 11:29:47 +0100 | [diff] [blame] | 342 | |
| 343 | if (!frame.delayed_by_retransmission()) |
Niels Möller | 2377588 | 2018-08-16 10:24:12 +0200 | [diff] [blame] | 344 | timing_->IncomingTimestamp(frame.Timestamp(), frame.ReceivedTime()); |
gnish | b2a318b | 2017-05-10 09:21:33 -0700 | [diff] [blame] | 345 | } |
| 346 | |
philipel | e7c891f | 2018-02-22 14:35:06 +0100 | [diff] [blame] | 347 | int64_t FrameBuffer::InsertFrame(std::unique_ptr<EncodedFrame> frame) { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 348 | TRACE_EVENT0("webrtc", "FrameBuffer::InsertFrame"); |
philipel | 93e451b | 2016-10-06 12:25:13 +0200 | [diff] [blame] | 349 | RTC_DCHECK(frame); |
philipel | a45102f | 2017-02-22 05:30:39 -0800 | [diff] [blame] | 350 | if (stats_callback_) |
ilnik | 6d5b4d6 | 2017-08-30 03:32:14 -0700 | [diff] [blame] | 351 | stats_callback_->OnCompleteFrame(frame->is_keyframe(), frame->size(), |
| 352 | frame->contentType()); |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 353 | const VideoLayerFrameId& id = frame->id; |
tommi | 0a73564 | 2017-03-14 06:23:57 -0700 | [diff] [blame] | 354 | |
| 355 | rtc::CritScope lock(&crit_); |
philipel | 29f730e | 2017-03-15 08:10:08 -0700 | [diff] [blame] | 356 | |
philipel | 1610f94 | 2017-12-12 13:58:31 +0100 | [diff] [blame] | 357 | int64_t last_continuous_picture_id = |
Ilya Nikolaevskiy | 6551faf | 2019-01-10 15:16:47 +0100 | [diff] [blame] | 358 | !last_continuous_frame_ ? -1 : last_continuous_frame_->picture_id; |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 359 | |
philipel | 112adf9 | 2017-06-15 09:06:21 -0700 | [diff] [blame] | 360 | if (!ValidReferences(*frame)) { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 361 | RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) (" |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 362 | << id.picture_id << ":" |
| 363 | << static_cast<int>(id.spatial_layer) |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 364 | << ") has invalid frame references, dropping frame."; |
philipel | 112adf9 | 2017-06-15 09:06:21 -0700 | [diff] [blame] | 365 | return last_continuous_picture_id; |
| 366 | } |
| 367 | |
Ilya Nikolaevskiy | 6551faf | 2019-01-10 15:16:47 +0100 | [diff] [blame] | 368 | if (frames_.size() >= kMaxFramesBuffered) { |
philipel | 9771c50 | 2018-03-02 11:06:27 +0100 | [diff] [blame] | 369 | if (frame->is_keyframe()) { |
| 370 | RTC_LOG(LS_WARNING) << "Inserting keyframe (picture_id:spatial_id) (" |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 371 | << id.picture_id << ":" |
| 372 | << static_cast<int>(id.spatial_layer) |
philipel | 9771c50 | 2018-03-02 11:06:27 +0100 | [diff] [blame] | 373 | << ") but buffer is full, clearing" |
| 374 | << " buffer and inserting the frame."; |
| 375 | ClearFramesAndHistory(); |
| 376 | } else { |
| 377 | RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) (" |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 378 | << id.picture_id << ":" |
| 379 | << static_cast<int>(id.spatial_layer) |
philipel | 9771c50 | 2018-03-02 11:06:27 +0100 | [diff] [blame] | 380 | << ") could not be inserted due to the frame " |
| 381 | << "buffer being full, dropping frame."; |
| 382 | return last_continuous_picture_id; |
| 383 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 384 | } |
| 385 | |
Ilya Nikolaevskiy | 1371784 | 2019-01-14 13:24:22 +0100 | [diff] [blame] | 386 | auto last_decoded_frame = decoded_frames_history_.GetLastDecodedFrameId(); |
| 387 | auto last_decoded_frame_timestamp = |
| 388 | decoded_frames_history_.GetLastDecodedFrameTimestamp(); |
| 389 | if (last_decoded_frame && id <= *last_decoded_frame) { |
| 390 | if (AheadOf(frame->Timestamp(), *last_decoded_frame_timestamp) && |
philipel | 3042c2d | 2017-08-18 04:55:02 -0700 | [diff] [blame] | 391 | frame->is_keyframe()) { |
philipel | fcc6006 | 2017-01-18 05:35:20 -0800 | [diff] [blame] | 392 | // If this frame has a newer timestamp but an earlier picture id then we |
| 393 | // assume there has been a jump in the picture id due to some encoder |
| 394 | // reconfiguration or some other reason. Even though this is not according |
| 395 | // to spec we can still continue to decode from this frame if it is a |
| 396 | // keyframe. |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 397 | RTC_LOG(LS_WARNING) |
| 398 | << "A jump in picture id was detected, clearing buffer."; |
philipel | fcc6006 | 2017-01-18 05:35:20 -0800 | [diff] [blame] | 399 | ClearFramesAndHistory(); |
| 400 | last_continuous_picture_id = -1; |
| 401 | } else { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 402 | RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) (" |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 403 | << id.picture_id << ":" |
| 404 | << static_cast<int>(id.spatial_layer) |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 405 | << ") inserted after frame (" |
Ilya Nikolaevskiy | 1371784 | 2019-01-14 13:24:22 +0100 | [diff] [blame] | 406 | << last_decoded_frame->picture_id << ":" |
| 407 | << static_cast<int>(last_decoded_frame->spatial_layer) |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 408 | << ") was handed off for decoding, dropping frame."; |
philipel | fcc6006 | 2017-01-18 05:35:20 -0800 | [diff] [blame] | 409 | return last_continuous_picture_id; |
| 410 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 411 | } |
| 412 | |
philipel | 146a48b | 2017-04-20 04:04:38 -0700 | [diff] [blame] | 413 | // Test if inserting this frame would cause the order of the frames to become |
| 414 | // ambiguous (covering more than half the interval of 2^16). This can happen |
| 415 | // when the picture id make large jumps mid stream. |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 416 | if (!frames_.empty() && id < frames_.begin()->first && |
| 417 | frames_.rbegin()->first < id) { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 418 | RTC_LOG(LS_WARNING) |
| 419 | << "A jump in picture id was detected, clearing buffer."; |
philipel | 146a48b | 2017-04-20 04:04:38 -0700 | [diff] [blame] | 420 | ClearFramesAndHistory(); |
| 421 | last_continuous_picture_id = -1; |
| 422 | } |
| 423 | |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 424 | auto info = frames_.emplace(id, FrameInfo()).first; |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 425 | |
philipel | 93e451b | 2016-10-06 12:25:13 +0200 | [diff] [blame] | 426 | if (info->second.frame) { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 427 | RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) (" |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 428 | << id.picture_id << ":" |
| 429 | << static_cast<int>(id.spatial_layer) |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 430 | << ") already inserted, dropping frame."; |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 431 | return last_continuous_picture_id; |
| 432 | } |
| 433 | |
philipel | 93e451b | 2016-10-06 12:25:13 +0200 | [diff] [blame] | 434 | if (!UpdateFrameInfoWithIncomingFrame(*frame, info)) |
| 435 | return last_continuous_picture_id; |
gnish | b2a318b | 2017-05-10 09:21:33 -0700 | [diff] [blame] | 436 | UpdatePlayoutDelays(*frame); |
philipel | 0a9f6de | 2018-02-28 11:29:47 +0100 | [diff] [blame] | 437 | |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 438 | info->second.frame = std::move(frame); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 439 | |
| 440 | if (info->second.num_missing_continuous == 0) { |
| 441 | info->second.continuous = true; |
| 442 | PropagateContinuity(info); |
Ilya Nikolaevskiy | 6551faf | 2019-01-10 15:16:47 +0100 | [diff] [blame] | 443 | last_continuous_picture_id = last_continuous_frame_->picture_id; |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 444 | |
| 445 | // Since we now have new continuous frames there might be a better frame |
| 446 | // to return from NextFrame. Signal that thread so that it again can choose |
| 447 | // which frame to return. |
tommi | 0a73564 | 2017-03-14 06:23:57 -0700 | [diff] [blame] | 448 | new_continuous_frame_event_.Set(); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 449 | } |
| 450 | |
| 451 | return last_continuous_picture_id; |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 452 | } |
| 453 | |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 454 | void FrameBuffer::PropagateContinuity(FrameMap::iterator start) { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 455 | TRACE_EVENT0("webrtc", "FrameBuffer::PropagateContinuity"); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 456 | RTC_DCHECK(start->second.continuous); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 457 | |
| 458 | std::queue<FrameMap::iterator> continuous_frames; |
| 459 | continuous_frames.push(start); |
| 460 | |
| 461 | // A simple BFS to traverse continuous frames. |
| 462 | while (!continuous_frames.empty()) { |
| 463 | auto frame = continuous_frames.front(); |
| 464 | continuous_frames.pop(); |
| 465 | |
Ilya Nikolaevskiy | 6551faf | 2019-01-10 15:16:47 +0100 | [diff] [blame] | 466 | if (!last_continuous_frame_ || *last_continuous_frame_ < frame->first) { |
| 467 | last_continuous_frame_ = frame->first; |
| 468 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 469 | |
| 470 | // Loop through all dependent frames, and if that frame no longer has |
| 471 | // any unfulfilled dependencies then that frame is continuous as well. |
Elad Alon | 69321dd | 2019-01-10 15:02:54 +0100 | [diff] [blame] | 472 | for (size_t d = 0; d < frame->second.dependent_frames.size(); ++d) { |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 473 | auto frame_ref = frames_.find(frame->second.dependent_frames[d]); |
philipel | 112adf9 | 2017-06-15 09:06:21 -0700 | [diff] [blame] | 474 | RTC_DCHECK(frame_ref != frames_.end()); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 475 | |
philipel | 112adf9 | 2017-06-15 09:06:21 -0700 | [diff] [blame] | 476 | // TODO(philipel): Look into why we've seen this happen. |
| 477 | if (frame_ref != frames_.end()) { |
| 478 | --frame_ref->second.num_missing_continuous; |
| 479 | if (frame_ref->second.num_missing_continuous == 0) { |
| 480 | frame_ref->second.continuous = true; |
| 481 | continuous_frames.push(frame_ref); |
| 482 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 483 | } |
| 484 | } |
| 485 | } |
| 486 | } |
| 487 | |
| 488 | void FrameBuffer::PropagateDecodability(const FrameInfo& info) { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 489 | TRACE_EVENT0("webrtc", "FrameBuffer::PropagateDecodability"); |
Elad Alon | 69321dd | 2019-01-10 15:02:54 +0100 | [diff] [blame] | 490 | for (size_t d = 0; d < info.dependent_frames.size(); ++d) { |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 491 | auto ref_info = frames_.find(info.dependent_frames[d]); |
philipel | 93e451b | 2016-10-06 12:25:13 +0200 | [diff] [blame] | 492 | RTC_DCHECK(ref_info != frames_.end()); |
tommi | e95b78b | 2017-05-14 07:23:11 -0700 | [diff] [blame] | 493 | // TODO(philipel): Look into why we've seen this happen. |
| 494 | if (ref_info != frames_.end()) { |
| 495 | RTC_DCHECK_GT(ref_info->second.num_missing_decodable, 0U); |
| 496 | --ref_info->second.num_missing_decodable; |
| 497 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 498 | } |
| 499 | } |
| 500 | |
philipel | e7c891f | 2018-02-22 14:35:06 +0100 | [diff] [blame] | 501 | bool FrameBuffer::UpdateFrameInfoWithIncomingFrame(const EncodedFrame& frame, |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 502 | FrameMap::iterator info) { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 503 | TRACE_EVENT0("webrtc", "FrameBuffer::UpdateFrameInfoWithIncomingFrame"); |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 504 | const VideoLayerFrameId& id = frame.id; |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 505 | |
Ilya Nikolaevskiy | 1371784 | 2019-01-14 13:24:22 +0100 | [diff] [blame] | 506 | auto last_decoded_frame = decoded_frames_history_.GetLastDecodedFrameId(); |
| 507 | RTC_DCHECK(!last_decoded_frame || *last_decoded_frame < info->first); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 508 | |
philipel | 798b282 | 2018-06-11 13:10:14 +0200 | [diff] [blame] | 509 | // In this function we determine how many missing dependencies this |frame| |
| 510 | // has to become continuous/decodable. If a frame that this |frame| depend |
| 511 | // on has already been decoded then we can ignore that dependency since it has |
| 512 | // already been fulfilled. |
| 513 | // |
| 514 | // For all other frames we will register a backwards reference to this |frame| |
| 515 | // so that |num_missing_continuous| and |num_missing_decodable| can be |
| 516 | // decremented as frames become continuous/are decoded. |
| 517 | struct Dependency { |
| 518 | VideoLayerFrameId id; |
| 519 | bool continuous; |
| 520 | }; |
| 521 | std::vector<Dependency> not_yet_fulfilled_dependencies; |
| 522 | |
| 523 | // Find all dependencies that have not yet been fulfilled. |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 524 | for (size_t i = 0; i < frame.num_references; ++i) { |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 525 | VideoLayerFrameId ref_key(frame.references[i], frame.id.spatial_layer); |
philipel | 798b282 | 2018-06-11 13:10:14 +0200 | [diff] [blame] | 526 | // Does |frame| depend on a frame earlier than the last decoded one? |
Ilya Nikolaevskiy | 1371784 | 2019-01-14 13:24:22 +0100 | [diff] [blame] | 527 | if (last_decoded_frame && ref_key <= *last_decoded_frame) { |
philipel | 798b282 | 2018-06-11 13:10:14 +0200 | [diff] [blame] | 528 | // Was that frame decoded? If not, this |frame| will never become |
| 529 | // decodable. |
Ilya Nikolaevskiy | 1371784 | 2019-01-14 13:24:22 +0100 | [diff] [blame] | 530 | if (!decoded_frames_history_.WasDecoded(ref_key)) { |
philipel | 65e1f94 | 2017-07-24 08:26:53 -0700 | [diff] [blame] | 531 | int64_t now_ms = clock_->TimeInMilliseconds(); |
| 532 | if (last_log_non_decoded_ms_ + kLogNonDecodedIntervalMs < now_ms) { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 533 | RTC_LOG(LS_WARNING) |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 534 | << "Frame with (picture_id:spatial_id) (" << id.picture_id << ":" |
| 535 | << static_cast<int>(id.spatial_layer) |
philipel | 65e1f94 | 2017-07-24 08:26:53 -0700 | [diff] [blame] | 536 | << ") depends on a non-decoded frame more previous than" |
| 537 | << " the last decoded frame, dropping frame."; |
| 538 | last_log_non_decoded_ms_ = now_ms; |
| 539 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 540 | return false; |
| 541 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 542 | } else { |
Ilya Nikolaevskiy | 6551faf | 2019-01-10 15:16:47 +0100 | [diff] [blame] | 543 | auto ref_info = frames_.find(ref_key); |
philipel | 798b282 | 2018-06-11 13:10:14 +0200 | [diff] [blame] | 544 | bool ref_continuous = |
| 545 | ref_info != frames_.end() && ref_info->second.continuous; |
| 546 | not_yet_fulfilled_dependencies.push_back({ref_key, ref_continuous}); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 547 | } |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 548 | } |
| 549 | |
philipel | 798b282 | 2018-06-11 13:10:14 +0200 | [diff] [blame] | 550 | // Does |frame| depend on the lower spatial layer? |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 551 | if (frame.inter_layer_predicted) { |
philipel | 0fa82a6 | 2018-03-19 15:34:53 +0100 | [diff] [blame] | 552 | VideoLayerFrameId ref_key(frame.id.picture_id, frame.id.spatial_layer - 1); |
philipel | 798b282 | 2018-06-11 13:10:14 +0200 | [diff] [blame] | 553 | auto ref_info = frames_.find(ref_key); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 554 | |
Ilya Nikolaevskiy | 6551faf | 2019-01-10 15:16:47 +0100 | [diff] [blame] | 555 | bool lower_layer_decoded = |
Ilya Nikolaevskiy | 1371784 | 2019-01-14 13:24:22 +0100 | [diff] [blame] | 556 | last_decoded_frame && *last_decoded_frame == ref_key; |
philipel | 798b282 | 2018-06-11 13:10:14 +0200 | [diff] [blame] | 557 | bool lower_layer_continuous = |
Ilya Nikolaevskiy | 6551faf | 2019-01-10 15:16:47 +0100 | [diff] [blame] | 558 | lower_layer_decoded || |
| 559 | (ref_info != frames_.end() && ref_info->second.continuous); |
philipel | 798b282 | 2018-06-11 13:10:14 +0200 | [diff] [blame] | 560 | |
| 561 | if (!lower_layer_continuous || !lower_layer_decoded) { |
| 562 | not_yet_fulfilled_dependencies.push_back( |
| 563 | {ref_key, lower_layer_continuous}); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 564 | } |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 565 | } |
| 566 | |
philipel | 798b282 | 2018-06-11 13:10:14 +0200 | [diff] [blame] | 567 | info->second.num_missing_continuous = not_yet_fulfilled_dependencies.size(); |
| 568 | info->second.num_missing_decodable = not_yet_fulfilled_dependencies.size(); |
| 569 | |
| 570 | for (const Dependency& dep : not_yet_fulfilled_dependencies) { |
| 571 | if (dep.continuous) |
| 572 | --info->second.num_missing_continuous; |
| 573 | |
Elad Alon | 69321dd | 2019-01-10 15:02:54 +0100 | [diff] [blame] | 574 | frames_[dep.id].dependent_frames.push_back(id); |
philipel | 798b282 | 2018-06-11 13:10:14 +0200 | [diff] [blame] | 575 | } |
philipel | 93e451b | 2016-10-06 12:25:13 +0200 | [diff] [blame] | 576 | |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 577 | return true; |
| 578 | } |
| 579 | |
philipel | be74270 | 2016-11-30 01:31:40 -0800 | [diff] [blame] | 580 | void FrameBuffer::UpdateJitterDelay() { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 581 | TRACE_EVENT0("webrtc", "FrameBuffer::UpdateJitterDelay"); |
philipel | a45102f | 2017-02-22 05:30:39 -0800 | [diff] [blame] | 582 | if (!stats_callback_) |
| 583 | return; |
philipel | be74270 | 2016-11-30 01:31:40 -0800 | [diff] [blame] | 584 | |
philipel | a45102f | 2017-02-22 05:30:39 -0800 | [diff] [blame] | 585 | int decode_ms; |
| 586 | int max_decode_ms; |
| 587 | int current_delay_ms; |
| 588 | int target_delay_ms; |
| 589 | int jitter_buffer_ms; |
| 590 | int min_playout_delay_ms; |
| 591 | int render_delay_ms; |
| 592 | if (timing_->GetTimings(&decode_ms, &max_decode_ms, ¤t_delay_ms, |
| 593 | &target_delay_ms, &jitter_buffer_ms, |
| 594 | &min_playout_delay_ms, &render_delay_ms)) { |
| 595 | stats_callback_->OnFrameBufferTimingsUpdated( |
| 596 | decode_ms, max_decode_ms, current_delay_ms, target_delay_ms, |
| 597 | jitter_buffer_ms, min_playout_delay_ms, render_delay_ms); |
philipel | be74270 | 2016-11-30 01:31:40 -0800 | [diff] [blame] | 598 | } |
philipel | 266f0a4 | 2016-11-28 08:49:07 -0800 | [diff] [blame] | 599 | } |
| 600 | |
ilnik | 2edc684 | 2017-07-06 03:06:50 -0700 | [diff] [blame] | 601 | void FrameBuffer::UpdateTimingFrameInfo() { |
| 602 | TRACE_EVENT0("webrtc", "FrameBuffer::UpdateTimingFrameInfo"); |
Danil Chapovalov | 0040b66 | 2018-06-18 10:48:16 +0200 | [diff] [blame] | 603 | absl::optional<TimingFrameInfo> info = timing_->GetTimingFrameInfo(); |
philipel | 9718711 | 2018-03-23 10:43:21 +0100 | [diff] [blame] | 604 | if (info && stats_callback_) |
ilnik | 2edc684 | 2017-07-06 03:06:50 -0700 | [diff] [blame] | 605 | stats_callback_->OnTimingFrameInfoUpdated(*info); |
| 606 | } |
| 607 | |
philipel | fcc6006 | 2017-01-18 05:35:20 -0800 | [diff] [blame] | 608 | void FrameBuffer::ClearFramesAndHistory() { |
ilnik | 2edc684 | 2017-07-06 03:06:50 -0700 | [diff] [blame] | 609 | TRACE_EVENT0("webrtc", "FrameBuffer::ClearFramesAndHistory"); |
philipel | fcc6006 | 2017-01-18 05:35:20 -0800 | [diff] [blame] | 610 | frames_.clear(); |
Ilya Nikolaevskiy | 6551faf | 2019-01-10 15:16:47 +0100 | [diff] [blame] | 611 | last_continuous_frame_.reset(); |
Ilya Nikolaevskiy | 5546aef | 2018-12-04 15:54:52 +0100 | [diff] [blame] | 612 | frames_to_decode_.clear(); |
Ilya Nikolaevskiy | 1371784 | 2019-01-14 13:24:22 +0100 | [diff] [blame] | 613 | decoded_frames_history_.Clear(); |
philipel | fcc6006 | 2017-01-18 05:35:20 -0800 | [diff] [blame] | 614 | } |
| 615 | |
Ilya Nikolaevskiy | 5546aef | 2018-12-04 15:54:52 +0100 | [diff] [blame] | 616 | EncodedFrame* FrameBuffer::CombineAndDeleteFrames( |
| 617 | const std::vector<EncodedFrame*>& frames) const { |
| 618 | RTC_DCHECK(!frames.empty()); |
Sergey Silkin | 61832dd | 2018-12-20 14:32:14 +0100 | [diff] [blame] | 619 | EncodedFrame* first_frame = frames[0]; |
| 620 | EncodedFrame* last_frame = frames.back(); |
Ilya Nikolaevskiy | 5546aef | 2018-12-04 15:54:52 +0100 | [diff] [blame] | 621 | size_t total_length = 0; |
| 622 | for (size_t i = 0; i < frames.size(); ++i) { |
| 623 | total_length += frames[i]->size(); |
| 624 | } |
Sergey Silkin | 61832dd | 2018-12-20 14:32:14 +0100 | [diff] [blame] | 625 | first_frame->VerifyAndAllocate(total_length); |
| 626 | |
| 627 | // Spatial index of combined frame is set equal to spatial index of its top |
| 628 | // spatial layer. |
| 629 | first_frame->SetSpatialIndex(last_frame->id.spatial_layer); |
| 630 | first_frame->id.spatial_layer = last_frame->id.spatial_layer; |
| 631 | |
| 632 | first_frame->video_timing_mutable()->network2_timestamp_ms = |
| 633 | last_frame->video_timing().network2_timestamp_ms; |
| 634 | first_frame->video_timing_mutable()->receive_finish_ms = |
| 635 | last_frame->video_timing().receive_finish_ms; |
| 636 | |
Ilya Nikolaevskiy | 5546aef | 2018-12-04 15:54:52 +0100 | [diff] [blame] | 637 | // Append all remaining frames to the first one. |
Niels Möller | 9c84390 | 2019-01-11 10:21:35 +0100 | [diff] [blame] | 638 | uint8_t* buffer = first_frame->data() + first_frame->size(); |
Ilya Nikolaevskiy | 5546aef | 2018-12-04 15:54:52 +0100 | [diff] [blame] | 639 | for (size_t i = 1; i < frames.size(); ++i) { |
Sergey Silkin | 61832dd | 2018-12-20 14:32:14 +0100 | [diff] [blame] | 640 | EncodedFrame* next_frame = frames[i]; |
Niels Möller | 9c84390 | 2019-01-11 10:21:35 +0100 | [diff] [blame] | 641 | memcpy(buffer, next_frame->data(), next_frame->size()); |
Sergey Silkin | 61832dd | 2018-12-20 14:32:14 +0100 | [diff] [blame] | 642 | buffer += next_frame->size(); |
| 643 | delete next_frame; |
Ilya Nikolaevskiy | 5546aef | 2018-12-04 15:54:52 +0100 | [diff] [blame] | 644 | } |
Sergey Silkin | 61832dd | 2018-12-20 14:32:14 +0100 | [diff] [blame] | 645 | first_frame->set_size(total_length); |
| 646 | return first_frame; |
Ilya Nikolaevskiy | 5546aef | 2018-12-04 15:54:52 +0100 | [diff] [blame] | 647 | } |
| 648 | |
Niels Möller | be682d4 | 2018-03-27 08:31:45 +0200 | [diff] [blame] | 649 | FrameBuffer::FrameInfo::FrameInfo() = default; |
| 650 | FrameBuffer::FrameInfo::FrameInfo(FrameInfo&&) = default; |
| 651 | FrameBuffer::FrameInfo::~FrameInfo() = default; |
| 652 | |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 653 | } // namespace video_coding |
| 654 | } // namespace webrtc |