blob: 4aaf12c8a14eaeee568828ce2e383766fae73421 [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:25 +00001/*
mikhal@webrtc.orga2031d52012-07-31 15:53:44 +00002 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
niklase@google.com470e71d2011-07-07 08:21:25 +00003 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "modules/video_coding/generic_decoder.h"
tommi5b7fc8c2017-07-05 16:45:57 -070012
Yves Gerey3e707812018-11-28 16:47:49 +010013#include <stddef.h>
Jonas Olssona4d87372019-07-05 19:08:33 +020014
ilnik2edc6842017-07-06 03:06:50 -070015#include <algorithm>
Johannes Kron111e9812020-10-26 13:54:40 +010016#include <cmath>
ilnik2edc6842017-07-06 03:06:50 -070017
Yves Gerey3e707812018-11-28 16:47:49 +010018#include "api/video/video_timing.h"
19#include "modules/video_coding/include/video_error_codes.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020020#include "rtc_base/checks.h"
21#include "rtc_base/logging.h"
Johannes Kron7ddea572019-09-11 12:00:22 +020022#include "rtc_base/thread.h"
Steve Anton10542f22019-01-11 09:11:00 -080023#include "rtc_base/time_utils.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020024#include "rtc_base/trace_event.h"
25#include "system_wrappers/include/clock.h"
niklase@google.com470e71d2011-07-07 08:21:25 +000026
27namespace webrtc {
28
Jonas Orelande02f9ee2022-03-25 12:43:14 +010029VCMDecodedFrameCallback::VCMDecodedFrameCallback(
30 VCMTiming* timing,
31 Clock* clock,
Jonas Orelande62c2f22022-03-29 11:04:48 +020032 const FieldTrialsView& field_trials)
Johannes Kron7ddea572019-09-11 12:00:22 +020033 : _clock(clock),
34 _timing(timing),
35 _timestampMap(kDecoderFrameMemoryLength),
Evan Shrubsolef6adc642022-04-26 11:10:21 +020036 _extra_decode_time("t", absl::nullopt) {
ilnik04f4d122017-06-19 07:18:55 -070037 ntp_offset_ =
38 _clock->CurrentNtpInMilliseconds() - _clock->TimeInMilliseconds();
Johannes Kron7ddea572019-09-11 12:00:22 +020039
40 ParseFieldTrial({&_extra_decode_time},
Jonas Orelande02f9ee2022-03-25 12:43:14 +010041 field_trials.Lookup("WebRTC-SlowDownDecoder"));
ilnik04f4d122017-06-19 07:18:55 -070042}
niklase@google.com470e71d2011-07-07 08:21:25 +000043
Yves Gerey665174f2018-06-19 15:03:05 +020044VCMDecodedFrameCallback::~VCMDecodedFrameCallback() {}
niklase@google.com470e71d2011-07-07 08:21:25 +000045
stefan@webrtc.org06887ae2011-10-10 14:17:46 +000046void VCMDecodedFrameCallback::SetUserReceiveCallback(
philipel9d3ab612015-12-21 04:12:39 -080047 VCMReceiveCallback* receiveCallback) {
Sebastian Janssonc01367d2019-04-08 15:20:44 +020048 RTC_DCHECK(construction_thread_.IsCurrent());
tommid0a71ba2017-03-14 04:16:20 -070049 RTC_DCHECK((!_receiveCallback && receiveCallback) ||
50 (_receiveCallback && !receiveCallback));
philipel9d3ab612015-12-21 04:12:39 -080051 _receiveCallback = receiveCallback;
niklase@google.com470e71d2011-07-07 08:21:25 +000052}
53
philipel9d3ab612015-12-21 04:12:39 -080054VCMReceiveCallback* VCMDecodedFrameCallback::UserReceiveCallback() {
tommid0a71ba2017-03-14 04:16:20 -070055 // Called on the decode thread via VCMCodecDataBase::GetDecoder.
56 // The callback must always have been set before this happens.
57 RTC_DCHECK(_receiveCallback);
philipel9d3ab612015-12-21 04:12:39 -080058 return _receiveCallback;
wuchengli@chromium.org0d94c2f2013-08-12 14:20:49 +000059}
60
Miguel Casas-Sanchez47650702015-05-29 17:21:40 -070061int32_t VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage) {
Tommi553c8692020-05-05 15:35:45 +020062 // This function may be called on the decode TaskQueue, but may also be called
63 // on an OS provided queue such as on iOS (see e.g. b/153465112).
Per327d8ba2015-11-10 14:00:27 +010064 return Decoded(decodedImage, -1);
65}
66
67int32_t VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage,
68 int64_t decode_time_ms) {
sakalcc452e12017-02-09 04:53:45 -080069 Decoded(decodedImage,
Danil Chapovalov0040b662018-06-18 10:48:16 +020070 decode_time_ms >= 0 ? absl::optional<int32_t>(decode_time_ms)
71 : absl::nullopt,
72 absl::nullopt);
sakalcc452e12017-02-09 04:53:45 -080073 return WEBRTC_VIDEO_CODEC_OK;
74}
75
76void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage,
Danil Chapovalov0040b662018-06-18 10:48:16 +020077 absl::optional<int32_t> decode_time_ms,
78 absl::optional<uint8_t> qp) {
Johannes Kron7ddea572019-09-11 12:00:22 +020079 // Wait some extra time to simulate a slow decoder.
80 if (_extra_decode_time) {
81 rtc::Thread::SleepMs(_extra_decode_time->ms());
82 }
83
tommid0a71ba2017-03-14 04:16:20 -070084 RTC_DCHECK(_receiveCallback) << "Callback must not be null at this point";
philipel9d3ab612015-12-21 04:12:39 -080085 TRACE_EVENT_INSTANT1("webrtc", "VCMDecodedFrameCallback::Decoded",
86 "timestamp", decodedImage.timestamp());
87 // TODO(holmer): We should improve this so that we can handle multiple
88 // callbacks from one call to Decode().
Johannes Kronb6b782d2021-03-03 14:39:44 +010089 absl::optional<VCMFrameInformation> frameInfo;
Johannes Kron111e9812020-10-26 13:54:40 +010090 int timestamp_map_size = 0;
Johannes Kronfc5d2762021-04-09 16:03:22 +020091 int dropped_frames = 0;
Lu Liu352314a2018-02-21 19:38:59 +000092 {
Markus Handell6deec382020-07-07 12:17:12 +020093 MutexLock lock(&lock_);
Johannes Kronfc5d2762021-04-09 16:03:22 +020094 int initial_timestamp_map_size = _timestampMap.Size();
Lu Liu352314a2018-02-21 19:38:59 +000095 frameInfo = _timestampMap.Pop(decodedImage.timestamp());
Johannes Kron111e9812020-10-26 13:54:40 +010096 timestamp_map_size = _timestampMap.Size();
Johannes Kronfc5d2762021-04-09 16:03:22 +020097 // _timestampMap.Pop() erases all frame upto the specified timestamp and
98 // return the frame info for this timestamp if it exists. Thus, the
99 // difference in the _timestampMap size before and after Pop() will show
100 // internally dropped frames.
101 dropped_frames =
102 initial_timestamp_map_size - timestamp_map_size - (frameInfo ? 1 : 0);
103 }
104
105 if (dropped_frames > 0) {
106 _receiveCallback->OnDroppedFrames(dropped_frames);
Lu Liu352314a2018-02-21 19:38:59 +0000107 }
philipel9d3ab612015-12-21 04:12:39 -0800108
Johannes Kronb6b782d2021-03-03 14:39:44 +0100109 if (!frameInfo) {
Mirko Bonadei675513b2017-11-09 11:09:25 +0100110 RTC_LOG(LS_WARNING) << "Too many frames backed up in the decoder, dropping "
Johannes Kronac82bd32021-06-17 10:50:56 +0200111 "frame with timestamp "
112 << decodedImage.timestamp();
sakalcc452e12017-02-09 04:53:45 -0800113 return;
philipel9d3ab612015-12-21 04:12:39 -0800114 }
115
Ilya Nikolaevskiy2ebf5232019-05-13 16:13:36 +0200116 decodedImage.set_ntp_time_ms(frameInfo->ntp_time_ms);
Chen Xingf00bf422019-06-20 10:05:55 +0200117 decodedImage.set_packet_infos(frameInfo->packet_infos);
Ilya Nikolaevskiy2ebf5232019-05-13 16:13:36 +0200118 decodedImage.set_rotation(frameInfo->rotation);
119
Evan Shrubsolef6adc642022-04-26 11:10:21 +0200120 absl::optional<int> max_composition_delay_in_frames =
121 _timing->MaxCompositionDelayInFrames();
122 if (max_composition_delay_in_frames) {
123 // Subtract frames that are in flight.
124 *max_composition_delay_in_frames -= timestamp_map_size;
125 *max_composition_delay_in_frames =
126 std::max(0, *max_composition_delay_in_frames);
127 decodedImage.set_max_composition_delay_in_frames(
128 max_composition_delay_in_frames);
Johannes Kron111e9812020-10-26 13:54:40 +0100129 }
130
Johannes Kron05f84872020-01-16 14:09:33 +0100131 RTC_DCHECK(frameInfo->decodeStart);
philipel85ddb232020-10-02 14:46:14 +0200132 const Timestamp now = _clock->CurrentTime();
133 const TimeDelta decode_time = decode_time_ms
134 ? TimeDelta::Millis(*decode_time_ms)
135 : now - *frameInfo->decodeStart;
Evan Shrubsoled6cdf802022-03-02 15:13:55 +0100136 _timing->StopDecodeTimer(decode_time, now);
philipel85ddb232020-10-02 14:46:14 +0200137 decodedImage.set_processing_time(
138 {*frameInfo->decodeStart, *frameInfo->decodeStart + decode_time});
philipel9d3ab612015-12-21 04:12:39 -0800139
ilnik04f4d122017-06-19 07:18:55 -0700140 // Report timing information.
Benjamin Wright3f10ca82018-12-07 03:45:19 -0800141 TimingFrameInfo timing_frame_info;
Ilya Nikolaevskiyb6c462d2018-06-05 15:21:32 +0200142 if (frameInfo->timing.flags != VideoSendTiming::kInvalid) {
ilnik2edc6842017-07-06 03:06:50 -0700143 int64_t capture_time_ms = decodedImage.ntp_time_ms() - ntp_offset_;
ilnik04f4d122017-06-19 07:18:55 -0700144 // Convert remote timestamps to local time from ntp timestamps.
145 frameInfo->timing.encode_start_ms -= ntp_offset_;
146 frameInfo->timing.encode_finish_ms -= ntp_offset_;
147 frameInfo->timing.packetization_finish_ms -= ntp_offset_;
148 frameInfo->timing.pacer_exit_ms -= ntp_offset_;
149 frameInfo->timing.network_timestamp_ms -= ntp_offset_;
150 frameInfo->timing.network2_timestamp_ms -= ntp_offset_;
ilnik2edc6842017-07-06 03:06:50 -0700151
152 int64_t sender_delta_ms = 0;
153 if (decodedImage.ntp_time_ms() < 0) {
154 // Sender clock is not estimated yet. Make sure that sender times are all
155 // negative to indicate that. Yet they still should be relatively correct.
156 sender_delta_ms =
157 std::max({capture_time_ms, frameInfo->timing.encode_start_ms,
158 frameInfo->timing.encode_finish_ms,
159 frameInfo->timing.packetization_finish_ms,
160 frameInfo->timing.pacer_exit_ms,
161 frameInfo->timing.network_timestamp_ms,
162 frameInfo->timing.network2_timestamp_ms}) +
163 1;
164 }
165
ilnik2edc6842017-07-06 03:06:50 -0700166 timing_frame_info.capture_time_ms = capture_time_ms - sender_delta_ms;
167 timing_frame_info.encode_start_ms =
168 frameInfo->timing.encode_start_ms - sender_delta_ms;
169 timing_frame_info.encode_finish_ms =
170 frameInfo->timing.encode_finish_ms - sender_delta_ms;
171 timing_frame_info.packetization_finish_ms =
172 frameInfo->timing.packetization_finish_ms - sender_delta_ms;
173 timing_frame_info.pacer_exit_ms =
174 frameInfo->timing.pacer_exit_ms - sender_delta_ms;
175 timing_frame_info.network_timestamp_ms =
176 frameInfo->timing.network_timestamp_ms - sender_delta_ms;
177 timing_frame_info.network2_timestamp_ms =
178 frameInfo->timing.network2_timestamp_ms - sender_delta_ms;
ilnik04f4d122017-06-19 07:18:55 -0700179 }
180
Benjamin Wright3f10ca82018-12-07 03:45:19 -0800181 timing_frame_info.flags = frameInfo->timing.flags;
Johannes Kron05f84872020-01-16 14:09:33 +0100182 timing_frame_info.decode_start_ms = frameInfo->decodeStart->ms();
183 timing_frame_info.decode_finish_ms = now.ms();
Benjamin Wright3f10ca82018-12-07 03:45:19 -0800184 timing_frame_info.render_time_ms = frameInfo->renderTimeMs;
185 timing_frame_info.rtp_timestamp = decodedImage.timestamp();
186 timing_frame_info.receive_start_ms = frameInfo->timing.receive_start_ms;
187 timing_frame_info.receive_finish_ms = frameInfo->timing.receive_finish_ms;
188 _timing->SetTimingFrameInfo(timing_frame_info);
189
Yves Gerey665174f2018-06-19 15:03:05 +0200190 decodedImage.set_timestamp_us(frameInfo->renderTimeMs *
191 rtc::kNumMicrosecsPerMillisec);
philipel85ddb232020-10-02 14:46:14 +0200192 _receiveCallback->FrameToRender(decodedImage, qp, decode_time.ms(),
Johannes Kronbfd343b2019-07-01 10:07:50 +0200193 frameInfo->content_type);
niklase@google.com470e71d2011-07-07 08:21:25 +0000194}
195
Peter Boströmb7d9a972015-12-18 16:01:11 +0100196void VCMDecodedFrameCallback::OnDecoderImplementationName(
197 const char* implementation_name) {
tommid0a71ba2017-03-14 04:16:20 -0700198 _receiveCallback->OnDecoderImplementationName(implementation_name);
Peter Boströmb7d9a972015-12-18 16:01:11 +0100199}
200
pbos1968d3f2015-09-28 08:52:18 -0700201void VCMDecodedFrameCallback::Map(uint32_t timestamp,
Johannes Kronb6b782d2021-03-03 14:39:44 +0100202 const VCMFrameInformation& frameInfo) {
Johannes Kronfc5d2762021-04-09 16:03:22 +0200203 int dropped_frames = 0;
204 {
205 MutexLock lock(&lock_);
206 int initial_size = _timestampMap.Size();
207 _timestampMap.Add(timestamp, frameInfo);
Artem Titovdcd7fc72021-08-09 13:02:57 +0200208 // If no frame is dropped, the new size should be `initial_size` + 1
Johannes Kronfc5d2762021-04-09 16:03:22 +0200209 dropped_frames = (initial_size + 1) - _timestampMap.Size();
210 }
211 if (dropped_frames > 0) {
212 _receiveCallback->OnDroppedFrames(dropped_frames);
213 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000214}
215
Johannes Kronfc5d2762021-04-09 16:03:22 +0200216void VCMDecodedFrameCallback::ClearTimestampMap() {
217 int dropped_frames = 0;
218 {
219 MutexLock lock(&lock_);
220 dropped_frames = _timestampMap.Size();
221 _timestampMap.Clear();
Lu Liu352314a2018-02-21 19:38:59 +0000222 }
Johannes Kronfc5d2762021-04-09 16:03:22 +0200223 if (dropped_frames > 0) {
224 _receiveCallback->OnDroppedFrames(dropped_frames);
225 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000226}
227
Danil Chapovalov7b78a312021-08-06 12:30:02 +0200228VCMGenericDecoder::VCMGenericDecoder(VideoDecoder* decoder)
Peter Boström187db632015-12-01 17:20:01 +0100229 : _callback(NULL),
tommi5b7fc8c2017-07-05 16:45:57 -0700230 decoder_(decoder),
tommi5b7fc8c2017-07-05 16:45:57 -0700231 _last_keyframe_content_type(VideoContentType::UNSPECIFIED) {
232 RTC_DCHECK(decoder_);
233}
niklase@google.com470e71d2011-07-07 08:21:25 +0000234
tommi5b7fc8c2017-07-05 16:45:57 -0700235VCMGenericDecoder::~VCMGenericDecoder() {
236 decoder_->Release();
tommi5b7fc8c2017-07-05 16:45:57 -0700237}
niklase@google.com470e71d2011-07-07 08:21:25 +0000238
Danil Chapovalov355b8d22021-08-13 16:50:37 +0200239bool VCMGenericDecoder::Configure(const VideoDecoder::Settings& settings) {
240 TRACE_EVENT0("webrtc", "VCMGenericDecoder::Configure");
niklase@google.com470e71d2011-07-07 08:21:25 +0000241
Danil Chapovalov355b8d22021-08-13 16:50:37 +0200242 bool ok = decoder_->Configure(settings);
Erik Språngc12f6252021-01-13 21:49:59 +0100243 decoder_info_ = decoder_->GetDecoderInfo();
244 RTC_LOG(LS_INFO) << "Decoder implementation: " << decoder_info_.ToString();
245 if (_callback) {
246 _callback->OnDecoderImplementationName(
247 decoder_info_.implementation_name.c_str());
248 }
Danil Chapovalov355b8d22021-08-13 16:50:37 +0200249 return ok;
niklase@google.com470e71d2011-07-07 08:21:25 +0000250}
251
Johannes Kron05f84872020-01-16 14:09:33 +0100252int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, Timestamp now) {
Yves Gerey665174f2018-06-19 15:03:05 +0200253 TRACE_EVENT1("webrtc", "VCMGenericDecoder::Decode", "timestamp",
Niels Möller23775882018-08-16 10:24:12 +0200254 frame.Timestamp());
Johannes Kronb6b782d2021-03-03 14:39:44 +0100255 VCMFrameInformation frame_info;
256 frame_info.decodeStart = now;
257 frame_info.renderTimeMs = frame.RenderTimeMs();
258 frame_info.rotation = frame.rotation();
259 frame_info.timing = frame.video_timing();
260 frame_info.ntp_time_ms = frame.EncodedImage().ntp_time_ms_;
261 frame_info.packet_infos = frame.PacketInfos();
Ilya Nikolaevskiy2ebf5232019-05-13 16:13:36 +0200262
Yves Gerey665174f2018-06-19 15:03:05 +0200263 // Set correctly only for key frames. Thus, use latest key frame
264 // content type. If the corresponding key frame was lost, decode will fail
265 // and content type will be ignored.
Niels Möller8f7ce222019-03-21 15:43:58 +0100266 if (frame.FrameType() == VideoFrameType::kVideoFrameKey) {
Johannes Kronb6b782d2021-03-03 14:39:44 +0100267 frame_info.content_type = frame.contentType();
Yves Gerey665174f2018-06-19 15:03:05 +0200268 _last_keyframe_content_type = frame.contentType();
269 } else {
Johannes Kronb6b782d2021-03-03 14:39:44 +0100270 frame_info.content_type = _last_keyframe_content_type;
Yves Gerey665174f2018-06-19 15:03:05 +0200271 }
Johannes Kronb6b782d2021-03-03 14:39:44 +0100272 _callback->Map(frame.Timestamp(), frame_info);
niklase@google.com470e71d2011-07-07 08:21:25 +0000273
Yves Gerey665174f2018-06-19 15:03:05 +0200274 int32_t ret = decoder_->Decode(frame.EncodedImage(), frame.MissingFrame(),
Niels Möller7aacdd92019-03-25 09:11:40 +0100275 frame.RenderTimeMs());
Erik Språngc12f6252021-01-13 21:49:59 +0100276 VideoDecoder::DecoderInfo decoder_info = decoder_->GetDecoderInfo();
277 if (decoder_info != decoder_info_) {
Ilya Nikolaevskiy43c108b2020-05-15 12:24:29 +0200278 RTC_LOG(LS_INFO) << "Changed decoder implementation to: "
Erik Språngc12f6252021-01-13 21:49:59 +0100279 << decoder_info.ToString();
Erik Språngc80f9552021-03-12 16:36:49 +0100280 decoder_info_ = decoder_info;
Erik Språngc12f6252021-01-13 21:49:59 +0100281 _callback->OnDecoderImplementationName(
282 decoder_info.implementation_name.empty()
283 ? "unknown"
284 : decoder_info.implementation_name.c_str());
Ilya Nikolaevskiy43c108b2020-05-15 12:24:29 +0200285 }
Yves Gerey665174f2018-06-19 15:03:05 +0200286 if (ret < WEBRTC_VIDEO_CODEC_OK) {
287 RTC_LOG(LS_WARNING) << "Failed to decode frame with timestamp "
Niels Möller23775882018-08-16 10:24:12 +0200288 << frame.Timestamp() << ", error code: " << ret;
Johannes Kronfc5d2762021-04-09 16:03:22 +0200289 _callback->ClearTimestampMap();
Niels Möller9cccf852018-12-04 11:01:26 +0100290 } else if (ret == WEBRTC_VIDEO_CODEC_NO_OUTPUT) {
Johannes Kronfc5d2762021-04-09 16:03:22 +0200291 // No output.
292 _callback->ClearTimestampMap();
Yves Gerey665174f2018-06-19 15:03:05 +0200293 }
294 return ret;
guidouc3372582017-04-04 07:16:21 -0700295}
niklase@google.com470e71d2011-07-07 08:21:25 +0000296
Peter Boström187db632015-12-01 17:20:01 +0100297int32_t VCMGenericDecoder::RegisterDecodeCompleteCallback(
298 VCMDecodedFrameCallback* callback) {
299 _callback = callback;
Erik Språngc12f6252021-01-13 21:49:59 +0100300 int32_t ret = decoder_->RegisterDecodeCompleteCallback(callback);
301 if (callback && !decoder_info_.implementation_name.empty()) {
302 callback->OnDecoderImplementationName(
303 decoder_info_.implementation_name.c_str());
304 }
305 return ret;
niklase@google.com470e71d2011-07-07 08:21:25 +0000306}
307
philipel9d3ab612015-12-21 04:12:39 -0800308} // namespace webrtc