philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 11 | #include "modules/video_coding/frame_buffer2.h" |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 12 | |
| 13 | #include <algorithm> |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 14 | #include <cstring> |
| 15 | #include <queue> |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 16 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 17 | #include "modules/video_coding/include/video_coding_defines.h" |
| 18 | #include "modules/video_coding/jitter_estimator.h" |
| 19 | #include "modules/video_coding/timing.h" |
| 20 | #include "rtc_base/checks.h" |
| 21 | #include "rtc_base/logging.h" |
| 22 | #include "rtc_base/trace_event.h" |
| 23 | #include "system_wrappers/include/clock.h" |
philipel | 707f278 | 2017-10-02 14:10:28 +0200 | [diff] [blame] | 24 | #include "system_wrappers/include/field_trial.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 25 | #include "system_wrappers/include/metrics.h" |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 26 | |
| 27 | namespace webrtc { |
| 28 | namespace video_coding { |
| 29 | |
| 30 | namespace { |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 31 | // Max number of frames the buffer will hold. |
| 32 | constexpr int kMaxFramesBuffered = 600; |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 33 | |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 34 | // Max number of decoded frame info that will be saved. |
philipel | fd5a20f | 2016-11-15 00:57:57 -0800 | [diff] [blame] | 35 | constexpr int kMaxFramesHistory = 50; |
philipel | 65e1f94 | 2017-07-24 08:26:53 -0700 | [diff] [blame] | 36 | |
| 37 | constexpr int64_t kLogNonDecodedIntervalMs = 5000; |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 38 | } // namespace |
| 39 | |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 40 | FrameBuffer::FrameBuffer(Clock* clock, |
| 41 | VCMJitterEstimator* jitter_estimator, |
philipel | a45102f | 2017-02-22 05:30:39 -0800 | [diff] [blame] | 42 | VCMTiming* timing, |
| 43 | VCMReceiveStatisticsCallback* stats_callback) |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 44 | : clock_(clock), |
tommi | 0a73564 | 2017-03-14 06:23:57 -0700 | [diff] [blame] | 45 | new_continuous_frame_event_(false, false), |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 46 | jitter_estimator_(jitter_estimator), |
| 47 | timing_(timing), |
philipel | 4f6cd6a | 2016-08-03 10:59:32 +0200 | [diff] [blame] | 48 | inter_frame_delay_(clock_->TimeInMilliseconds()), |
gnish | b2a318b | 2017-05-10 09:21:33 -0700 | [diff] [blame] | 49 | last_decoded_frame_timestamp_(0), |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 50 | last_decoded_frame_it_(frames_.end()), |
| 51 | last_continuous_frame_it_(frames_.end()), |
| 52 | num_frames_history_(0), |
| 53 | num_frames_buffered_(0), |
philipel | 29f730e | 2017-03-15 08:10:08 -0700 | [diff] [blame] | 54 | stopped_(false), |
philipel | a45102f | 2017-02-22 05:30:39 -0800 | [diff] [blame] | 55 | protection_mode_(kProtectionNack), |
philipel | 65e1f94 | 2017-07-24 08:26:53 -0700 | [diff] [blame] | 56 | stats_callback_(stats_callback), |
| 57 | last_log_non_decoded_ms_(-kLogNonDecodedIntervalMs) {} |
philipel | 266f0a4 | 2016-11-28 08:49:07 -0800 | [diff] [blame] | 58 | |
philipel | a45102f | 2017-02-22 05:30:39 -0800 | [diff] [blame] | 59 | FrameBuffer::~FrameBuffer() {} |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 60 | |
philipel | 7556282 | 2016-09-05 10:57:41 +0200 | [diff] [blame] | 61 | FrameBuffer::ReturnReason FrameBuffer::NextFrame( |
| 62 | int64_t max_wait_time_ms, |
philipel | e7c891f | 2018-02-22 14:35:06 +0100 | [diff] [blame] | 63 | std::unique_ptr<EncodedFrame>* frame_out, |
philipel | 3042c2d | 2017-08-18 04:55:02 -0700 | [diff] [blame] | 64 | bool keyframe_required) { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 65 | TRACE_EVENT0("webrtc", "FrameBuffer::NextFrame"); |
philipel | 1c05625 | 2017-01-31 09:53:12 -0800 | [diff] [blame] | 66 | int64_t latest_return_time_ms = |
| 67 | clock_->TimeInMilliseconds() + max_wait_time_ms; |
philipel | 504c47d | 2016-06-30 17:33:02 +0200 | [diff] [blame] | 68 | int64_t wait_ms = max_wait_time_ms; |
philipel | 29f730e | 2017-03-15 08:10:08 -0700 | [diff] [blame] | 69 | int64_t now_ms = 0; |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 70 | |
| 71 | do { |
philipel | 29f730e | 2017-03-15 08:10:08 -0700 | [diff] [blame] | 72 | now_ms = clock_->TimeInMilliseconds(); |
philipel | 504c47d | 2016-06-30 17:33:02 +0200 | [diff] [blame] | 73 | { |
| 74 | rtc::CritScope lock(&crit_); |
tommi | 0a73564 | 2017-03-14 06:23:57 -0700 | [diff] [blame] | 75 | new_continuous_frame_event_.Reset(); |
philipel | 29f730e | 2017-03-15 08:10:08 -0700 | [diff] [blame] | 76 | if (stopped_) |
| 77 | return kStopped; |
| 78 | |
| 79 | wait_ms = max_wait_time_ms; |
| 80 | |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 81 | // Need to hold |crit_| in order to use |frames_|, therefore we |
| 82 | // set it here in the loop instead of outside the loop in order to not |
| 83 | // acquire the lock unnecesserily. |
philipel | 1c05625 | 2017-01-31 09:53:12 -0800 | [diff] [blame] | 84 | next_frame_it_ = frames_.end(); |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 85 | |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 86 | // |frame_it| points to the first frame after the |
| 87 | // |last_decoded_frame_it_|. |
| 88 | auto frame_it = frames_.end(); |
| 89 | if (last_decoded_frame_it_ == frames_.end()) { |
| 90 | frame_it = frames_.begin(); |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 91 | } else { |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 92 | frame_it = last_decoded_frame_it_; |
| 93 | ++frame_it; |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 94 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 95 | |
| 96 | // |continuous_end_it| points to the first frame after the |
| 97 | // |last_continuous_frame_it_|. |
| 98 | auto continuous_end_it = last_continuous_frame_it_; |
| 99 | if (continuous_end_it != frames_.end()) |
| 100 | ++continuous_end_it; |
| 101 | |
philipel | 146a48b | 2017-04-20 04:04:38 -0700 | [diff] [blame] | 102 | for (; frame_it != continuous_end_it && frame_it != frames_.end(); |
| 103 | ++frame_it) { |
philipel | 93e451b | 2016-10-06 12:25:13 +0200 | [diff] [blame] | 104 | if (!frame_it->second.continuous || |
| 105 | frame_it->second.num_missing_decodable > 0) { |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 106 | continue; |
philipel | 93e451b | 2016-10-06 12:25:13 +0200 | [diff] [blame] | 107 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 108 | |
philipel | e7c891f | 2018-02-22 14:35:06 +0100 | [diff] [blame] | 109 | EncodedFrame* frame = frame_it->second.frame.get(); |
philipel | 3042c2d | 2017-08-18 04:55:02 -0700 | [diff] [blame] | 110 | |
| 111 | if (keyframe_required && !frame->is_keyframe()) |
| 112 | continue; |
| 113 | |
philipel | 1c05625 | 2017-01-31 09:53:12 -0800 | [diff] [blame] | 114 | next_frame_it_ = frame_it; |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 115 | if (frame->RenderTime() == -1) |
| 116 | frame->SetRenderTime(timing_->RenderTimeMs(frame->timestamp, now_ms)); |
| 117 | wait_ms = timing_->MaxWaitingTime(frame->RenderTime(), now_ms); |
| 118 | |
| 119 | // This will cause the frame buffer to prefer high framerate rather |
| 120 | // than high resolution in the case of the decoder not decoding fast |
| 121 | // enough and the stream has multiple spatial and temporal layers. |
| 122 | if (wait_ms == 0) |
| 123 | continue; |
| 124 | |
| 125 | break; |
| 126 | } |
| 127 | } // rtc::Critscope lock(&crit_); |
| 128 | |
philipel | 1c05625 | 2017-01-31 09:53:12 -0800 | [diff] [blame] | 129 | wait_ms = std::min<int64_t>(wait_ms, latest_return_time_ms - now_ms); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 130 | wait_ms = std::max<int64_t>(wait_ms, 0); |
tommi | 0a73564 | 2017-03-14 06:23:57 -0700 | [diff] [blame] | 131 | } while (new_continuous_frame_event_.Wait(wait_ms)); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 132 | |
philipel | 29f730e | 2017-03-15 08:10:08 -0700 | [diff] [blame] | 133 | { |
| 134 | rtc::CritScope lock(&crit_); |
| 135 | now_ms = clock_->TimeInMilliseconds(); |
| 136 | if (next_frame_it_ != frames_.end()) { |
philipel | e7c891f | 2018-02-22 14:35:06 +0100 | [diff] [blame] | 137 | std::unique_ptr<EncodedFrame> frame = |
philipel | 29f730e | 2017-03-15 08:10:08 -0700 | [diff] [blame] | 138 | std::move(next_frame_it_->second.frame); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 139 | |
philipel | 29f730e | 2017-03-15 08:10:08 -0700 | [diff] [blame] | 140 | if (!frame->delayed_by_retransmission()) { |
| 141 | int64_t frame_delay; |
philipel | e075430 | 2017-01-25 08:56:23 -0800 | [diff] [blame] | 142 | |
philipel | 29f730e | 2017-03-15 08:10:08 -0700 | [diff] [blame] | 143 | if (inter_frame_delay_.CalculateDelay(frame->timestamp, &frame_delay, |
| 144 | frame->ReceivedTime())) { |
| 145 | jitter_estimator_->UpdateEstimate(frame_delay, frame->size()); |
| 146 | } |
| 147 | |
| 148 | float rtt_mult = protection_mode_ == kProtectionNackFEC ? 0.0 : 1.0; |
| 149 | timing_->SetJitterDelay(jitter_estimator_->GetJitterEstimate(rtt_mult)); |
| 150 | timing_->UpdateCurrentDelay(frame->RenderTime(), now_ms); |
philipel | e21be1d | 2017-09-25 06:37:12 -0700 | [diff] [blame] | 151 | } else { |
philipel | 707f278 | 2017-10-02 14:10:28 +0200 | [diff] [blame] | 152 | if (webrtc::field_trial::IsEnabled("WebRTC-AddRttToPlayoutDelay")) |
| 153 | jitter_estimator_->FrameNacked(); |
philipel | e075430 | 2017-01-25 08:56:23 -0800 | [diff] [blame] | 154 | } |
| 155 | |
stefan | 95e9754 | 2017-05-23 09:52:18 -0700 | [diff] [blame] | 156 | // Gracefully handle bad RTP timestamps and render time issues. |
| 157 | if (HasBadRenderTiming(*frame, now_ms)) { |
| 158 | jitter_estimator_->Reset(); |
| 159 | timing_->Reset(); |
| 160 | frame->SetRenderTime(timing_->RenderTimeMs(frame->timestamp, now_ms)); |
| 161 | } |
| 162 | |
philipel | 29f730e | 2017-03-15 08:10:08 -0700 | [diff] [blame] | 163 | UpdateJitterDelay(); |
ilnik | 2edc684 | 2017-07-06 03:06:50 -0700 | [diff] [blame] | 164 | UpdateTimingFrameInfo(); |
philipel | 29f730e | 2017-03-15 08:10:08 -0700 | [diff] [blame] | 165 | PropagateDecodability(next_frame_it_->second); |
brandtr | 9078d8c | 2017-04-27 07:07:27 -0700 | [diff] [blame] | 166 | |
| 167 | // Sanity check for RTP timestamp monotonicity. |
| 168 | if (last_decoded_frame_it_ != frames_.end()) { |
| 169 | const FrameKey& last_decoded_frame_key = last_decoded_frame_it_->first; |
| 170 | const FrameKey& frame_key = next_frame_it_->first; |
| 171 | |
| 172 | const bool frame_is_higher_spatial_layer_of_last_decoded_frame = |
| 173 | last_decoded_frame_timestamp_ == frame->timestamp && |
| 174 | last_decoded_frame_key.picture_id == frame_key.picture_id && |
| 175 | last_decoded_frame_key.spatial_layer < frame_key.spatial_layer; |
| 176 | |
| 177 | if (AheadOrAt(last_decoded_frame_timestamp_, frame->timestamp) && |
| 178 | !frame_is_higher_spatial_layer_of_last_decoded_frame) { |
| 179 | // TODO(brandtr): Consider clearing the entire buffer when we hit |
| 180 | // these conditions. |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 181 | RTC_LOG(LS_WARNING) |
| 182 | << "Frame with (timestamp:picture_id:spatial_id) (" |
| 183 | << frame->timestamp << ":" << frame->picture_id << ":" |
| 184 | << static_cast<int>(frame->spatial_layer) << ")" |
| 185 | << " sent to decoder after frame with" |
| 186 | << " (timestamp:picture_id:spatial_id) (" |
| 187 | << last_decoded_frame_timestamp_ << ":" |
| 188 | << last_decoded_frame_key.picture_id << ":" |
| 189 | << static_cast<int>(last_decoded_frame_key.spatial_layer) << ")."; |
brandtr | 9078d8c | 2017-04-27 07:07:27 -0700 | [diff] [blame] | 190 | } |
| 191 | } |
| 192 | |
philipel | 29f730e | 2017-03-15 08:10:08 -0700 | [diff] [blame] | 193 | AdvanceLastDecodedFrame(next_frame_it_); |
| 194 | last_decoded_frame_timestamp_ = frame->timestamp; |
| 195 | *frame_out = std::move(frame); |
| 196 | return kFrameFound; |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 197 | } |
tommi | 0a73564 | 2017-03-14 06:23:57 -0700 | [diff] [blame] | 198 | } |
| 199 | |
| 200 | if (latest_return_time_ms - now_ms > 0) { |
philipel | 1c05625 | 2017-01-31 09:53:12 -0800 | [diff] [blame] | 201 | // If |next_frame_it_ == frames_.end()| and there is still time left, it |
| 202 | // means that the frame buffer was cleared as the thread in this function |
| 203 | // was waiting to acquire |crit_| in order to return. Wait for the |
| 204 | // remaining time and then return. |
| 205 | return NextFrame(latest_return_time_ms - now_ms, frame_out); |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 206 | } |
tommi | 0a73564 | 2017-03-14 06:23:57 -0700 | [diff] [blame] | 207 | |
| 208 | return kTimeout; |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 209 | } |
| 210 | |
philipel | e7c891f | 2018-02-22 14:35:06 +0100 | [diff] [blame] | 211 | bool FrameBuffer::HasBadRenderTiming(const EncodedFrame& frame, |
| 212 | int64_t now_ms) { |
stefan | 95e9754 | 2017-05-23 09:52:18 -0700 | [diff] [blame] | 213 | // Assume that render timing errors are due to changes in the video stream. |
| 214 | int64_t render_time_ms = frame.RenderTimeMs(); |
| 215 | const int64_t kMaxVideoDelayMs = 10000; |
| 216 | if (render_time_ms < 0) { |
| 217 | return true; |
| 218 | } |
| 219 | if (std::abs(render_time_ms - now_ms) > kMaxVideoDelayMs) { |
| 220 | int frame_delay = static_cast<int>(std::abs(render_time_ms - now_ms)); |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 221 | RTC_LOG(LS_WARNING) |
| 222 | << "A frame about to be decoded is out of the configured " |
| 223 | << "delay bounds (" << frame_delay << " > " << kMaxVideoDelayMs |
| 224 | << "). Resetting the video jitter buffer."; |
stefan | 95e9754 | 2017-05-23 09:52:18 -0700 | [diff] [blame] | 225 | return true; |
| 226 | } |
| 227 | if (static_cast<int>(timing_->TargetVideoDelay()) > kMaxVideoDelayMs) { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 228 | RTC_LOG(LS_WARNING) << "The video target delay has grown larger than " |
| 229 | << kMaxVideoDelayMs << " ms."; |
stefan | 95e9754 | 2017-05-23 09:52:18 -0700 | [diff] [blame] | 230 | return true; |
| 231 | } |
| 232 | return false; |
| 233 | } |
| 234 | |
philipel | 4f6cd6a | 2016-08-03 10:59:32 +0200 | [diff] [blame] | 235 | void FrameBuffer::SetProtectionMode(VCMVideoProtection mode) { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 236 | TRACE_EVENT0("webrtc", "FrameBuffer::SetProtectionMode"); |
philipel | 4f6cd6a | 2016-08-03 10:59:32 +0200 | [diff] [blame] | 237 | rtc::CritScope lock(&crit_); |
| 238 | protection_mode_ = mode; |
| 239 | } |
| 240 | |
philipel | 504c47d | 2016-06-30 17:33:02 +0200 | [diff] [blame] | 241 | void FrameBuffer::Start() { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 242 | TRACE_EVENT0("webrtc", "FrameBuffer::Start"); |
philipel | 29f730e | 2017-03-15 08:10:08 -0700 | [diff] [blame] | 243 | rtc::CritScope lock(&crit_); |
| 244 | stopped_ = false; |
philipel | 504c47d | 2016-06-30 17:33:02 +0200 | [diff] [blame] | 245 | } |
| 246 | |
| 247 | void FrameBuffer::Stop() { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 248 | TRACE_EVENT0("webrtc", "FrameBuffer::Stop"); |
philipel | 29f730e | 2017-03-15 08:10:08 -0700 | [diff] [blame] | 249 | rtc::CritScope lock(&crit_); |
| 250 | stopped_ = true; |
tommi | 0a73564 | 2017-03-14 06:23:57 -0700 | [diff] [blame] | 251 | new_continuous_frame_event_.Set(); |
philipel | 504c47d | 2016-06-30 17:33:02 +0200 | [diff] [blame] | 252 | } |
| 253 | |
philipel | e21be1d | 2017-09-25 06:37:12 -0700 | [diff] [blame] | 254 | void FrameBuffer::UpdateRtt(int64_t rtt_ms) { |
| 255 | rtc::CritScope lock(&crit_); |
| 256 | jitter_estimator_->UpdateRtt(rtt_ms); |
| 257 | } |
| 258 | |
philipel | e7c891f | 2018-02-22 14:35:06 +0100 | [diff] [blame] | 259 | bool FrameBuffer::ValidReferences(const EncodedFrame& frame) const { |
philipel | 3b3c9c4 | 2017-09-11 09:38:36 -0700 | [diff] [blame] | 260 | if (frame.picture_id < 0) |
| 261 | return false; |
| 262 | |
philipel | 112adf9 | 2017-06-15 09:06:21 -0700 | [diff] [blame] | 263 | for (size_t i = 0; i < frame.num_references; ++i) { |
philipel | 3b3c9c4 | 2017-09-11 09:38:36 -0700 | [diff] [blame] | 264 | if (frame.references[i] < 0 || frame.references[i] >= frame.picture_id) |
philipel | 112adf9 | 2017-06-15 09:06:21 -0700 | [diff] [blame] | 265 | return false; |
philipel | 3b3c9c4 | 2017-09-11 09:38:36 -0700 | [diff] [blame] | 266 | |
philipel | 112adf9 | 2017-06-15 09:06:21 -0700 | [diff] [blame] | 267 | for (size_t j = i + 1; j < frame.num_references; ++j) { |
| 268 | if (frame.references[i] == frame.references[j]) |
| 269 | return false; |
| 270 | } |
| 271 | } |
| 272 | |
| 273 | if (frame.inter_layer_predicted && frame.spatial_layer == 0) |
| 274 | return false; |
| 275 | |
| 276 | return true; |
| 277 | } |
| 278 | |
philipel | e7c891f | 2018-02-22 14:35:06 +0100 | [diff] [blame] | 279 | void FrameBuffer::UpdatePlayoutDelays(const EncodedFrame& frame) { |
gnish | b2a318b | 2017-05-10 09:21:33 -0700 | [diff] [blame] | 280 | TRACE_EVENT0("webrtc", "FrameBuffer::UpdatePlayoutDelays"); |
| 281 | PlayoutDelay playout_delay = frame.EncodedImage().playout_delay_; |
| 282 | if (playout_delay.min_ms >= 0) |
| 283 | timing_->set_min_playout_delay(playout_delay.min_ms); |
| 284 | |
| 285 | if (playout_delay.max_ms >= 0) |
| 286 | timing_->set_max_playout_delay(playout_delay.max_ms); |
| 287 | } |
| 288 | |
philipel | e7c891f | 2018-02-22 14:35:06 +0100 | [diff] [blame] | 289 | int64_t FrameBuffer::InsertFrame(std::unique_ptr<EncodedFrame> frame) { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 290 | TRACE_EVENT0("webrtc", "FrameBuffer::InsertFrame"); |
philipel | 93e451b | 2016-10-06 12:25:13 +0200 | [diff] [blame] | 291 | RTC_DCHECK(frame); |
philipel | a45102f | 2017-02-22 05:30:39 -0800 | [diff] [blame] | 292 | if (stats_callback_) |
ilnik | 6d5b4d6 | 2017-08-30 03:32:14 -0700 | [diff] [blame] | 293 | stats_callback_->OnCompleteFrame(frame->is_keyframe(), frame->size(), |
| 294 | frame->contentType()); |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 295 | FrameKey key(frame->picture_id, frame->spatial_layer); |
tommi | 0a73564 | 2017-03-14 06:23:57 -0700 | [diff] [blame] | 296 | |
| 297 | rtc::CritScope lock(&crit_); |
philipel | 29f730e | 2017-03-15 08:10:08 -0700 | [diff] [blame] | 298 | |
philipel | 1610f94 | 2017-12-12 13:58:31 +0100 | [diff] [blame] | 299 | int64_t last_continuous_picture_id = |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 300 | last_continuous_frame_it_ == frames_.end() |
| 301 | ? -1 |
| 302 | : last_continuous_frame_it_->first.picture_id; |
| 303 | |
philipel | 112adf9 | 2017-06-15 09:06:21 -0700 | [diff] [blame] | 304 | if (!ValidReferences(*frame)) { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 305 | RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) (" |
| 306 | << key.picture_id << ":" |
| 307 | << static_cast<int>(key.spatial_layer) |
| 308 | << ") has invalid frame references, dropping frame."; |
philipel | 112adf9 | 2017-06-15 09:06:21 -0700 | [diff] [blame] | 309 | return last_continuous_picture_id; |
| 310 | } |
| 311 | |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 312 | if (num_frames_buffered_ >= kMaxFramesBuffered) { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 313 | RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) (" |
| 314 | << key.picture_id << ":" |
| 315 | << static_cast<int>(key.spatial_layer) |
| 316 | << ") could not be inserted due to the frame " |
| 317 | << "buffer being full, dropping frame."; |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 318 | return last_continuous_picture_id; |
| 319 | } |
| 320 | |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 321 | if (last_decoded_frame_it_ != frames_.end() && |
philipel | f684269 | 2017-04-28 03:29:15 -0700 | [diff] [blame] | 322 | key <= last_decoded_frame_it_->first) { |
philipel | fcc6006 | 2017-01-18 05:35:20 -0800 | [diff] [blame] | 323 | if (AheadOf(frame->timestamp, last_decoded_frame_timestamp_) && |
philipel | 3042c2d | 2017-08-18 04:55:02 -0700 | [diff] [blame] | 324 | frame->is_keyframe()) { |
philipel | fcc6006 | 2017-01-18 05:35:20 -0800 | [diff] [blame] | 325 | // If this frame has a newer timestamp but an earlier picture id then we |
| 326 | // assume there has been a jump in the picture id due to some encoder |
| 327 | // reconfiguration or some other reason. Even though this is not according |
| 328 | // to spec we can still continue to decode from this frame if it is a |
| 329 | // keyframe. |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 330 | RTC_LOG(LS_WARNING) |
| 331 | << "A jump in picture id was detected, clearing buffer."; |
philipel | fcc6006 | 2017-01-18 05:35:20 -0800 | [diff] [blame] | 332 | ClearFramesAndHistory(); |
| 333 | last_continuous_picture_id = -1; |
| 334 | } else { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 335 | RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) (" |
| 336 | << key.picture_id << ":" |
| 337 | << static_cast<int>(key.spatial_layer) |
| 338 | << ") inserted after frame (" |
| 339 | << last_decoded_frame_it_->first.picture_id << ":" |
| 340 | << static_cast<int>( |
| 341 | last_decoded_frame_it_->first.spatial_layer) |
| 342 | << ") was handed off for decoding, dropping frame."; |
philipel | fcc6006 | 2017-01-18 05:35:20 -0800 | [diff] [blame] | 343 | return last_continuous_picture_id; |
| 344 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 345 | } |
| 346 | |
philipel | 146a48b | 2017-04-20 04:04:38 -0700 | [diff] [blame] | 347 | // Test if inserting this frame would cause the order of the frames to become |
| 348 | // ambiguous (covering more than half the interval of 2^16). This can happen |
| 349 | // when the picture id make large jumps mid stream. |
| 350 | if (!frames_.empty() && |
| 351 | key < frames_.begin()->first && |
| 352 | frames_.rbegin()->first < key) { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 353 | RTC_LOG(LS_WARNING) |
| 354 | << "A jump in picture id was detected, clearing buffer."; |
philipel | 146a48b | 2017-04-20 04:04:38 -0700 | [diff] [blame] | 355 | ClearFramesAndHistory(); |
| 356 | last_continuous_picture_id = -1; |
| 357 | } |
| 358 | |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 359 | auto info = frames_.insert(std::make_pair(key, FrameInfo())).first; |
| 360 | |
philipel | 93e451b | 2016-10-06 12:25:13 +0200 | [diff] [blame] | 361 | if (info->second.frame) { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 362 | RTC_LOG(LS_WARNING) << "Frame with (picture_id:spatial_id) (" |
| 363 | << key.picture_id << ":" |
| 364 | << static_cast<int>(key.spatial_layer) |
| 365 | << ") already inserted, dropping frame."; |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 366 | return last_continuous_picture_id; |
| 367 | } |
| 368 | |
philipel | 93e451b | 2016-10-06 12:25:13 +0200 | [diff] [blame] | 369 | if (!UpdateFrameInfoWithIncomingFrame(*frame, info)) |
| 370 | return last_continuous_picture_id; |
gnish | b2a318b | 2017-05-10 09:21:33 -0700 | [diff] [blame] | 371 | UpdatePlayoutDelays(*frame); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 372 | info->second.frame = std::move(frame); |
| 373 | ++num_frames_buffered_; |
| 374 | |
| 375 | if (info->second.num_missing_continuous == 0) { |
| 376 | info->second.continuous = true; |
| 377 | PropagateContinuity(info); |
| 378 | last_continuous_picture_id = last_continuous_frame_it_->first.picture_id; |
| 379 | |
| 380 | // Since we now have new continuous frames there might be a better frame |
| 381 | // to return from NextFrame. Signal that thread so that it again can choose |
| 382 | // which frame to return. |
tommi | 0a73564 | 2017-03-14 06:23:57 -0700 | [diff] [blame] | 383 | new_continuous_frame_event_.Set(); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 384 | } |
| 385 | |
| 386 | return last_continuous_picture_id; |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 387 | } |
| 388 | |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 389 | void FrameBuffer::PropagateContinuity(FrameMap::iterator start) { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 390 | TRACE_EVENT0("webrtc", "FrameBuffer::PropagateContinuity"); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 391 | RTC_DCHECK(start->second.continuous); |
| 392 | if (last_continuous_frame_it_ == frames_.end()) |
| 393 | last_continuous_frame_it_ = start; |
| 394 | |
| 395 | std::queue<FrameMap::iterator> continuous_frames; |
| 396 | continuous_frames.push(start); |
| 397 | |
| 398 | // A simple BFS to traverse continuous frames. |
| 399 | while (!continuous_frames.empty()) { |
| 400 | auto frame = continuous_frames.front(); |
| 401 | continuous_frames.pop(); |
| 402 | |
| 403 | if (last_continuous_frame_it_->first < frame->first) |
| 404 | last_continuous_frame_it_ = frame; |
| 405 | |
| 406 | // Loop through all dependent frames, and if that frame no longer has |
| 407 | // any unfulfilled dependencies then that frame is continuous as well. |
| 408 | for (size_t d = 0; d < frame->second.num_dependent_frames; ++d) { |
| 409 | auto frame_ref = frames_.find(frame->second.dependent_frames[d]); |
philipel | 112adf9 | 2017-06-15 09:06:21 -0700 | [diff] [blame] | 410 | RTC_DCHECK(frame_ref != frames_.end()); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 411 | |
philipel | 112adf9 | 2017-06-15 09:06:21 -0700 | [diff] [blame] | 412 | // TODO(philipel): Look into why we've seen this happen. |
| 413 | if (frame_ref != frames_.end()) { |
| 414 | --frame_ref->second.num_missing_continuous; |
| 415 | if (frame_ref->second.num_missing_continuous == 0) { |
| 416 | frame_ref->second.continuous = true; |
| 417 | continuous_frames.push(frame_ref); |
| 418 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 419 | } |
| 420 | } |
| 421 | } |
| 422 | } |
| 423 | |
| 424 | void FrameBuffer::PropagateDecodability(const FrameInfo& info) { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 425 | TRACE_EVENT0("webrtc", "FrameBuffer::PropagateDecodability"); |
tommi | e95b78b | 2017-05-14 07:23:11 -0700 | [diff] [blame] | 426 | RTC_CHECK(info.num_dependent_frames < FrameInfo::kMaxNumDependentFrames); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 427 | for (size_t d = 0; d < info.num_dependent_frames; ++d) { |
| 428 | auto ref_info = frames_.find(info.dependent_frames[d]); |
philipel | 93e451b | 2016-10-06 12:25:13 +0200 | [diff] [blame] | 429 | RTC_DCHECK(ref_info != frames_.end()); |
tommi | e95b78b | 2017-05-14 07:23:11 -0700 | [diff] [blame] | 430 | // TODO(philipel): Look into why we've seen this happen. |
| 431 | if (ref_info != frames_.end()) { |
| 432 | RTC_DCHECK_GT(ref_info->second.num_missing_decodable, 0U); |
| 433 | --ref_info->second.num_missing_decodable; |
| 434 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 435 | } |
| 436 | } |
| 437 | |
| 438 | void FrameBuffer::AdvanceLastDecodedFrame(FrameMap::iterator decoded) { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 439 | TRACE_EVENT0("webrtc", "FrameBuffer::AdvanceLastDecodedFrame"); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 440 | if (last_decoded_frame_it_ == frames_.end()) { |
| 441 | last_decoded_frame_it_ = frames_.begin(); |
| 442 | } else { |
| 443 | RTC_DCHECK(last_decoded_frame_it_->first < decoded->first); |
| 444 | ++last_decoded_frame_it_; |
| 445 | } |
| 446 | --num_frames_buffered_; |
| 447 | ++num_frames_history_; |
| 448 | |
| 449 | // First, delete non-decoded frames from the history. |
| 450 | while (last_decoded_frame_it_ != decoded) { |
| 451 | if (last_decoded_frame_it_->second.frame) |
| 452 | --num_frames_buffered_; |
| 453 | last_decoded_frame_it_ = frames_.erase(last_decoded_frame_it_); |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 454 | } |
| 455 | |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 456 | // Then remove old history if we have too much history saved. |
| 457 | if (num_frames_history_ > kMaxFramesHistory) { |
| 458 | frames_.erase(frames_.begin()); |
| 459 | --num_frames_history_; |
| 460 | } |
| 461 | } |
| 462 | |
philipel | e7c891f | 2018-02-22 14:35:06 +0100 | [diff] [blame] | 463 | bool FrameBuffer::UpdateFrameInfoWithIncomingFrame(const EncodedFrame& frame, |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 464 | FrameMap::iterator info) { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 465 | TRACE_EVENT0("webrtc", "FrameBuffer::UpdateFrameInfoWithIncomingFrame"); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 466 | FrameKey key(frame.picture_id, frame.spatial_layer); |
| 467 | info->second.num_missing_continuous = frame.num_references; |
| 468 | info->second.num_missing_decodable = frame.num_references; |
| 469 | |
| 470 | RTC_DCHECK(last_decoded_frame_it_ == frames_.end() || |
| 471 | last_decoded_frame_it_->first < info->first); |
| 472 | |
| 473 | // Check how many dependencies that have already been fulfilled. |
| 474 | for (size_t i = 0; i < frame.num_references; ++i) { |
| 475 | FrameKey ref_key(frame.references[i], frame.spatial_layer); |
| 476 | auto ref_info = frames_.find(ref_key); |
| 477 | |
| 478 | // Does |frame| depend on a frame earlier than the last decoded frame? |
| 479 | if (last_decoded_frame_it_ != frames_.end() && |
| 480 | ref_key <= last_decoded_frame_it_->first) { |
| 481 | if (ref_info == frames_.end()) { |
philipel | 65e1f94 | 2017-07-24 08:26:53 -0700 | [diff] [blame] | 482 | int64_t now_ms = clock_->TimeInMilliseconds(); |
| 483 | if (last_log_non_decoded_ms_ + kLogNonDecodedIntervalMs < now_ms) { |
Mirko Bonadei | 675513b | 2017-11-09 11:09:25 +0100 | [diff] [blame] | 484 | RTC_LOG(LS_WARNING) |
philipel | 65e1f94 | 2017-07-24 08:26:53 -0700 | [diff] [blame] | 485 | << "Frame with (picture_id:spatial_id) (" << key.picture_id << ":" |
| 486 | << static_cast<int>(key.spatial_layer) |
| 487 | << ") depends on a non-decoded frame more previous than" |
| 488 | << " the last decoded frame, dropping frame."; |
| 489 | last_log_non_decoded_ms_ = now_ms; |
| 490 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 491 | return false; |
| 492 | } |
| 493 | |
| 494 | --info->second.num_missing_continuous; |
| 495 | --info->second.num_missing_decodable; |
| 496 | } else { |
| 497 | if (ref_info == frames_.end()) |
| 498 | ref_info = frames_.insert(std::make_pair(ref_key, FrameInfo())).first; |
| 499 | |
| 500 | if (ref_info->second.continuous) |
| 501 | --info->second.num_missing_continuous; |
| 502 | |
| 503 | // Add backwards reference so |frame| can be updated when new |
| 504 | // frames are inserted or decoded. |
| 505 | ref_info->second.dependent_frames[ref_info->second.num_dependent_frames] = |
| 506 | key; |
tommi | e95b78b | 2017-05-14 07:23:11 -0700 | [diff] [blame] | 507 | RTC_DCHECK_LT(ref_info->second.num_dependent_frames, |
| 508 | (FrameInfo::kMaxNumDependentFrames - 1)); |
| 509 | // TODO(philipel): Look into why this could happen and handle |
| 510 | // appropriately. |
| 511 | if (ref_info->second.num_dependent_frames < |
| 512 | (FrameInfo::kMaxNumDependentFrames - 1)) { |
| 513 | ++ref_info->second.num_dependent_frames; |
| 514 | } |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 515 | } |
philipel | 93e451b | 2016-10-06 12:25:13 +0200 | [diff] [blame] | 516 | RTC_DCHECK_LE(ref_info->second.num_missing_continuous, |
| 517 | ref_info->second.num_missing_decodable); |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 518 | } |
| 519 | |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 520 | // Check if we have the lower spatial layer frame. |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 521 | if (frame.inter_layer_predicted) { |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 522 | ++info->second.num_missing_continuous; |
| 523 | ++info->second.num_missing_decodable; |
| 524 | |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 525 | FrameKey ref_key(frame.picture_id, frame.spatial_layer - 1); |
philipel | e0b2f15 | 2016-09-28 10:23:49 +0200 | [diff] [blame] | 526 | // Gets or create the FrameInfo for the referenced frame. |
| 527 | auto ref_info = frames_.insert(std::make_pair(ref_key, FrameInfo())).first; |
| 528 | if (ref_info->second.continuous) |
| 529 | --info->second.num_missing_continuous; |
| 530 | |
| 531 | if (ref_info == last_decoded_frame_it_) { |
| 532 | --info->second.num_missing_decodable; |
| 533 | } else { |
| 534 | ref_info->second.dependent_frames[ref_info->second.num_dependent_frames] = |
| 535 | key; |
| 536 | ++ref_info->second.num_dependent_frames; |
| 537 | } |
philipel | 93e451b | 2016-10-06 12:25:13 +0200 | [diff] [blame] | 538 | RTC_DCHECK_LE(ref_info->second.num_missing_continuous, |
| 539 | ref_info->second.num_missing_decodable); |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 540 | } |
| 541 | |
philipel | 93e451b | 2016-10-06 12:25:13 +0200 | [diff] [blame] | 542 | RTC_DCHECK_LE(info->second.num_missing_continuous, |
| 543 | info->second.num_missing_decodable); |
| 544 | |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 545 | return true; |
| 546 | } |
| 547 | |
philipel | be74270 | 2016-11-30 01:31:40 -0800 | [diff] [blame] | 548 | void FrameBuffer::UpdateJitterDelay() { |
tommi | db23ea6 | 2017-03-03 07:21:18 -0800 | [diff] [blame] | 549 | TRACE_EVENT0("webrtc", "FrameBuffer::UpdateJitterDelay"); |
philipel | a45102f | 2017-02-22 05:30:39 -0800 | [diff] [blame] | 550 | if (!stats_callback_) |
| 551 | return; |
philipel | be74270 | 2016-11-30 01:31:40 -0800 | [diff] [blame] | 552 | |
philipel | a45102f | 2017-02-22 05:30:39 -0800 | [diff] [blame] | 553 | int decode_ms; |
| 554 | int max_decode_ms; |
| 555 | int current_delay_ms; |
| 556 | int target_delay_ms; |
| 557 | int jitter_buffer_ms; |
| 558 | int min_playout_delay_ms; |
| 559 | int render_delay_ms; |
| 560 | if (timing_->GetTimings(&decode_ms, &max_decode_ms, ¤t_delay_ms, |
| 561 | &target_delay_ms, &jitter_buffer_ms, |
| 562 | &min_playout_delay_ms, &render_delay_ms)) { |
| 563 | stats_callback_->OnFrameBufferTimingsUpdated( |
| 564 | decode_ms, max_decode_ms, current_delay_ms, target_delay_ms, |
| 565 | jitter_buffer_ms, min_playout_delay_ms, render_delay_ms); |
philipel | be74270 | 2016-11-30 01:31:40 -0800 | [diff] [blame] | 566 | } |
philipel | 266f0a4 | 2016-11-28 08:49:07 -0800 | [diff] [blame] | 567 | } |
| 568 | |
ilnik | 2edc684 | 2017-07-06 03:06:50 -0700 | [diff] [blame] | 569 | void FrameBuffer::UpdateTimingFrameInfo() { |
| 570 | TRACE_EVENT0("webrtc", "FrameBuffer::UpdateTimingFrameInfo"); |
| 571 | rtc::Optional<TimingFrameInfo> info = timing_->GetTimingFrameInfo(); |
| 572 | if (info) |
| 573 | stats_callback_->OnTimingFrameInfoUpdated(*info); |
| 574 | } |
| 575 | |
philipel | fcc6006 | 2017-01-18 05:35:20 -0800 | [diff] [blame] | 576 | void FrameBuffer::ClearFramesAndHistory() { |
ilnik | 2edc684 | 2017-07-06 03:06:50 -0700 | [diff] [blame] | 577 | TRACE_EVENT0("webrtc", "FrameBuffer::ClearFramesAndHistory"); |
philipel | fcc6006 | 2017-01-18 05:35:20 -0800 | [diff] [blame] | 578 | frames_.clear(); |
| 579 | last_decoded_frame_it_ = frames_.end(); |
| 580 | last_continuous_frame_it_ = frames_.end(); |
philipel | 1c05625 | 2017-01-31 09:53:12 -0800 | [diff] [blame] | 581 | next_frame_it_ = frames_.end(); |
philipel | fcc6006 | 2017-01-18 05:35:20 -0800 | [diff] [blame] | 582 | num_frames_history_ = 0; |
| 583 | num_frames_buffered_ = 0; |
| 584 | } |
| 585 | |
philipel | be7a9e5 | 2016-05-19 12:19:35 +0200 | [diff] [blame] | 586 | } // namespace video_coding |
| 587 | } // namespace webrtc |