blob: 68eb1fc7e6d06bd0095aec5eb9b8cbc772ca4c89 [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:25 +00001/*
mikhal@webrtc.orga2031d52012-07-31 15:53:44 +00002 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
niklase@google.com470e71d2011-07-07 08:21:25 +00003 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "modules/video_coding/generic_decoder.h"
tommi5b7fc8c2017-07-05 16:45:57 -070012
Yves Gerey3e707812018-11-28 16:47:49 +010013#include <stddef.h>
Jonas Olssona4d87372019-07-05 19:08:33 +020014
ilnik2edc6842017-07-06 03:06:50 -070015#include <algorithm>
Johannes Kron111e9812020-10-26 13:54:40 +010016#include <cmath>
ilnik2edc6842017-07-06 03:06:50 -070017
Yves Gerey3e707812018-11-28 16:47:49 +010018#include "api/video/video_timing.h"
19#include "modules/video_coding/include/video_error_codes.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020020#include "rtc_base/checks.h"
21#include "rtc_base/logging.h"
Steve Anton10542f22019-01-11 09:11:00 -080022#include "rtc_base/time_utils.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "rtc_base/trace_event.h"
24#include "system_wrappers/include/clock.h"
niklase@google.com470e71d2011-07-07 08:21:25 +000025
26namespace webrtc {
27
Jonas Orelande02f9ee2022-03-25 12:43:14 +010028VCMDecodedFrameCallback::VCMDecodedFrameCallback(
29 VCMTiming* timing,
30 Clock* clock,
Jonas Orelande62c2f22022-03-29 11:04:48 +020031 const FieldTrialsView& field_trials)
Johannes Krona45bbfe2022-05-09 12:43:58 +020032 : _clock(clock), _timing(timing), _timestampMap(kDecoderFrameMemoryLength) {
ilnik04f4d122017-06-19 07:18:55 -070033 ntp_offset_ =
34 _clock->CurrentNtpInMilliseconds() - _clock->TimeInMilliseconds();
35}
niklase@google.com470e71d2011-07-07 08:21:25 +000036
Yves Gerey665174f2018-06-19 15:03:05 +020037VCMDecodedFrameCallback::~VCMDecodedFrameCallback() {}
niklase@google.com470e71d2011-07-07 08:21:25 +000038
stefan@webrtc.org06887ae2011-10-10 14:17:46 +000039void VCMDecodedFrameCallback::SetUserReceiveCallback(
philipel9d3ab612015-12-21 04:12:39 -080040 VCMReceiveCallback* receiveCallback) {
Sebastian Janssonc01367d2019-04-08 15:20:44 +020041 RTC_DCHECK(construction_thread_.IsCurrent());
tommid0a71ba2017-03-14 04:16:20 -070042 RTC_DCHECK((!_receiveCallback && receiveCallback) ||
43 (_receiveCallback && !receiveCallback));
philipel9d3ab612015-12-21 04:12:39 -080044 _receiveCallback = receiveCallback;
niklase@google.com470e71d2011-07-07 08:21:25 +000045}
46
philipel9d3ab612015-12-21 04:12:39 -080047VCMReceiveCallback* VCMDecodedFrameCallback::UserReceiveCallback() {
tommid0a71ba2017-03-14 04:16:20 -070048 // Called on the decode thread via VCMCodecDataBase::GetDecoder.
49 // The callback must always have been set before this happens.
50 RTC_DCHECK(_receiveCallback);
philipel9d3ab612015-12-21 04:12:39 -080051 return _receiveCallback;
wuchengli@chromium.org0d94c2f2013-08-12 14:20:49 +000052}
53
Miguel Casas-Sanchez47650702015-05-29 17:21:40 -070054int32_t VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage) {
Tommi553c8692020-05-05 15:35:45 +020055 // This function may be called on the decode TaskQueue, but may also be called
56 // on an OS provided queue such as on iOS (see e.g. b/153465112).
Per327d8ba2015-11-10 14:00:27 +010057 return Decoded(decodedImage, -1);
58}
59
60int32_t VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage,
61 int64_t decode_time_ms) {
sakalcc452e12017-02-09 04:53:45 -080062 Decoded(decodedImage,
Danil Chapovalov0040b662018-06-18 10:48:16 +020063 decode_time_ms >= 0 ? absl::optional<int32_t>(decode_time_ms)
64 : absl::nullopt,
65 absl::nullopt);
sakalcc452e12017-02-09 04:53:45 -080066 return WEBRTC_VIDEO_CODEC_OK;
67}
68
69void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage,
Danil Chapovalov0040b662018-06-18 10:48:16 +020070 absl::optional<int32_t> decode_time_ms,
71 absl::optional<uint8_t> qp) {
tommid0a71ba2017-03-14 04:16:20 -070072 RTC_DCHECK(_receiveCallback) << "Callback must not be null at this point";
philipel9d3ab612015-12-21 04:12:39 -080073 TRACE_EVENT_INSTANT1("webrtc", "VCMDecodedFrameCallback::Decoded",
74 "timestamp", decodedImage.timestamp());
75 // TODO(holmer): We should improve this so that we can handle multiple
76 // callbacks from one call to Decode().
Johannes Kronb6b782d2021-03-03 14:39:44 +010077 absl::optional<VCMFrameInformation> frameInfo;
Johannes Kron111e9812020-10-26 13:54:40 +010078 int timestamp_map_size = 0;
Johannes Kronfc5d2762021-04-09 16:03:22 +020079 int dropped_frames = 0;
Lu Liu352314a2018-02-21 19:38:59 +000080 {
Markus Handell6deec382020-07-07 12:17:12 +020081 MutexLock lock(&lock_);
Johannes Kronfc5d2762021-04-09 16:03:22 +020082 int initial_timestamp_map_size = _timestampMap.Size();
Lu Liu352314a2018-02-21 19:38:59 +000083 frameInfo = _timestampMap.Pop(decodedImage.timestamp());
Johannes Kron111e9812020-10-26 13:54:40 +010084 timestamp_map_size = _timestampMap.Size();
Johannes Kronfc5d2762021-04-09 16:03:22 +020085 // _timestampMap.Pop() erases all frame upto the specified timestamp and
86 // return the frame info for this timestamp if it exists. Thus, the
87 // difference in the _timestampMap size before and after Pop() will show
88 // internally dropped frames.
89 dropped_frames =
90 initial_timestamp_map_size - timestamp_map_size - (frameInfo ? 1 : 0);
91 }
92
93 if (dropped_frames > 0) {
94 _receiveCallback->OnDroppedFrames(dropped_frames);
Lu Liu352314a2018-02-21 19:38:59 +000095 }
philipel9d3ab612015-12-21 04:12:39 -080096
Johannes Kronb6b782d2021-03-03 14:39:44 +010097 if (!frameInfo) {
Mirko Bonadei675513b2017-11-09 11:09:25 +010098 RTC_LOG(LS_WARNING) << "Too many frames backed up in the decoder, dropping "
Johannes Kronac82bd32021-06-17 10:50:56 +020099 "frame with timestamp "
100 << decodedImage.timestamp();
sakalcc452e12017-02-09 04:53:45 -0800101 return;
philipel9d3ab612015-12-21 04:12:39 -0800102 }
103
Ilya Nikolaevskiy2ebf5232019-05-13 16:13:36 +0200104 decodedImage.set_ntp_time_ms(frameInfo->ntp_time_ms);
Chen Xingf00bf422019-06-20 10:05:55 +0200105 decodedImage.set_packet_infos(frameInfo->packet_infos);
Ilya Nikolaevskiy2ebf5232019-05-13 16:13:36 +0200106 decodedImage.set_rotation(frameInfo->rotation);
Johannes Kronbbf639e2022-06-15 12:27:23 +0200107 VideoFrame::RenderParameters render_parameters = _timing->RenderParameters();
108 if (render_parameters.max_composition_delay_in_frames) {
Evan Shrubsolef6adc642022-04-26 11:10:21 +0200109 // Subtract frames that are in flight.
Johannes Kronbbf639e2022-06-15 12:27:23 +0200110 render_parameters.max_composition_delay_in_frames =
111 std::max(0, *render_parameters.max_composition_delay_in_frames -
112 timestamp_map_size);
Johannes Kron111e9812020-10-26 13:54:40 +0100113 }
Johannes Kronbbf639e2022-06-15 12:27:23 +0200114 decodedImage.set_render_parameters(render_parameters);
Johannes Kron111e9812020-10-26 13:54:40 +0100115
Johannes Kron05f84872020-01-16 14:09:33 +0100116 RTC_DCHECK(frameInfo->decodeStart);
philipel85ddb232020-10-02 14:46:14 +0200117 const Timestamp now = _clock->CurrentTime();
118 const TimeDelta decode_time = decode_time_ms
119 ? TimeDelta::Millis(*decode_time_ms)
120 : now - *frameInfo->decodeStart;
Evan Shrubsoled6cdf802022-03-02 15:13:55 +0100121 _timing->StopDecodeTimer(decode_time, now);
philipel85ddb232020-10-02 14:46:14 +0200122 decodedImage.set_processing_time(
123 {*frameInfo->decodeStart, *frameInfo->decodeStart + decode_time});
philipel9d3ab612015-12-21 04:12:39 -0800124
ilnik04f4d122017-06-19 07:18:55 -0700125 // Report timing information.
Benjamin Wright3f10ca82018-12-07 03:45:19 -0800126 TimingFrameInfo timing_frame_info;
Ilya Nikolaevskiyb6c462d2018-06-05 15:21:32 +0200127 if (frameInfo->timing.flags != VideoSendTiming::kInvalid) {
ilnik2edc6842017-07-06 03:06:50 -0700128 int64_t capture_time_ms = decodedImage.ntp_time_ms() - ntp_offset_;
ilnik04f4d122017-06-19 07:18:55 -0700129 // Convert remote timestamps to local time from ntp timestamps.
130 frameInfo->timing.encode_start_ms -= ntp_offset_;
131 frameInfo->timing.encode_finish_ms -= ntp_offset_;
132 frameInfo->timing.packetization_finish_ms -= ntp_offset_;
133 frameInfo->timing.pacer_exit_ms -= ntp_offset_;
134 frameInfo->timing.network_timestamp_ms -= ntp_offset_;
135 frameInfo->timing.network2_timestamp_ms -= ntp_offset_;
ilnik2edc6842017-07-06 03:06:50 -0700136
137 int64_t sender_delta_ms = 0;
138 if (decodedImage.ntp_time_ms() < 0) {
139 // Sender clock is not estimated yet. Make sure that sender times are all
140 // negative to indicate that. Yet they still should be relatively correct.
141 sender_delta_ms =
142 std::max({capture_time_ms, frameInfo->timing.encode_start_ms,
143 frameInfo->timing.encode_finish_ms,
144 frameInfo->timing.packetization_finish_ms,
145 frameInfo->timing.pacer_exit_ms,
146 frameInfo->timing.network_timestamp_ms,
147 frameInfo->timing.network2_timestamp_ms}) +
148 1;
149 }
150
ilnik2edc6842017-07-06 03:06:50 -0700151 timing_frame_info.capture_time_ms = capture_time_ms - sender_delta_ms;
152 timing_frame_info.encode_start_ms =
153 frameInfo->timing.encode_start_ms - sender_delta_ms;
154 timing_frame_info.encode_finish_ms =
155 frameInfo->timing.encode_finish_ms - sender_delta_ms;
156 timing_frame_info.packetization_finish_ms =
157 frameInfo->timing.packetization_finish_ms - sender_delta_ms;
158 timing_frame_info.pacer_exit_ms =
159 frameInfo->timing.pacer_exit_ms - sender_delta_ms;
160 timing_frame_info.network_timestamp_ms =
161 frameInfo->timing.network_timestamp_ms - sender_delta_ms;
162 timing_frame_info.network2_timestamp_ms =
163 frameInfo->timing.network2_timestamp_ms - sender_delta_ms;
ilnik04f4d122017-06-19 07:18:55 -0700164 }
165
Benjamin Wright3f10ca82018-12-07 03:45:19 -0800166 timing_frame_info.flags = frameInfo->timing.flags;
Johannes Kron05f84872020-01-16 14:09:33 +0100167 timing_frame_info.decode_start_ms = frameInfo->decodeStart->ms();
168 timing_frame_info.decode_finish_ms = now.ms();
Benjamin Wright3f10ca82018-12-07 03:45:19 -0800169 timing_frame_info.render_time_ms = frameInfo->renderTimeMs;
170 timing_frame_info.rtp_timestamp = decodedImage.timestamp();
171 timing_frame_info.receive_start_ms = frameInfo->timing.receive_start_ms;
172 timing_frame_info.receive_finish_ms = frameInfo->timing.receive_finish_ms;
173 _timing->SetTimingFrameInfo(timing_frame_info);
174
Yves Gerey665174f2018-06-19 15:03:05 +0200175 decodedImage.set_timestamp_us(frameInfo->renderTimeMs *
176 rtc::kNumMicrosecsPerMillisec);
philipel85ddb232020-10-02 14:46:14 +0200177 _receiveCallback->FrameToRender(decodedImage, qp, decode_time.ms(),
Johannes Kronbfd343b2019-07-01 10:07:50 +0200178 frameInfo->content_type);
niklase@google.com470e71d2011-07-07 08:21:25 +0000179}
180
Peter Boströmb7d9a972015-12-18 16:01:11 +0100181void VCMDecodedFrameCallback::OnDecoderImplementationName(
182 const char* implementation_name) {
tommid0a71ba2017-03-14 04:16:20 -0700183 _receiveCallback->OnDecoderImplementationName(implementation_name);
Peter Boströmb7d9a972015-12-18 16:01:11 +0100184}
185
pbos1968d3f2015-09-28 08:52:18 -0700186void VCMDecodedFrameCallback::Map(uint32_t timestamp,
Johannes Kronb6b782d2021-03-03 14:39:44 +0100187 const VCMFrameInformation& frameInfo) {
Johannes Kronfc5d2762021-04-09 16:03:22 +0200188 int dropped_frames = 0;
189 {
190 MutexLock lock(&lock_);
191 int initial_size = _timestampMap.Size();
192 _timestampMap.Add(timestamp, frameInfo);
Artem Titovdcd7fc72021-08-09 13:02:57 +0200193 // If no frame is dropped, the new size should be `initial_size` + 1
Johannes Kronfc5d2762021-04-09 16:03:22 +0200194 dropped_frames = (initial_size + 1) - _timestampMap.Size();
195 }
196 if (dropped_frames > 0) {
197 _receiveCallback->OnDroppedFrames(dropped_frames);
198 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000199}
200
Johannes Kronfc5d2762021-04-09 16:03:22 +0200201void VCMDecodedFrameCallback::ClearTimestampMap() {
202 int dropped_frames = 0;
203 {
204 MutexLock lock(&lock_);
205 dropped_frames = _timestampMap.Size();
206 _timestampMap.Clear();
Lu Liu352314a2018-02-21 19:38:59 +0000207 }
Johannes Kronfc5d2762021-04-09 16:03:22 +0200208 if (dropped_frames > 0) {
209 _receiveCallback->OnDroppedFrames(dropped_frames);
210 }
niklase@google.com470e71d2011-07-07 08:21:25 +0000211}
212
Danil Chapovalov7b78a312021-08-06 12:30:02 +0200213VCMGenericDecoder::VCMGenericDecoder(VideoDecoder* decoder)
Peter Boström187db632015-12-01 17:20:01 +0100214 : _callback(NULL),
tommi5b7fc8c2017-07-05 16:45:57 -0700215 decoder_(decoder),
tommi5b7fc8c2017-07-05 16:45:57 -0700216 _last_keyframe_content_type(VideoContentType::UNSPECIFIED) {
217 RTC_DCHECK(decoder_);
218}
niklase@google.com470e71d2011-07-07 08:21:25 +0000219
tommi5b7fc8c2017-07-05 16:45:57 -0700220VCMGenericDecoder::~VCMGenericDecoder() {
221 decoder_->Release();
tommi5b7fc8c2017-07-05 16:45:57 -0700222}
niklase@google.com470e71d2011-07-07 08:21:25 +0000223
Danil Chapovalov355b8d22021-08-13 16:50:37 +0200224bool VCMGenericDecoder::Configure(const VideoDecoder::Settings& settings) {
225 TRACE_EVENT0("webrtc", "VCMGenericDecoder::Configure");
niklase@google.com470e71d2011-07-07 08:21:25 +0000226
Danil Chapovalov355b8d22021-08-13 16:50:37 +0200227 bool ok = decoder_->Configure(settings);
Erik Språngc12f6252021-01-13 21:49:59 +0100228 decoder_info_ = decoder_->GetDecoderInfo();
229 RTC_LOG(LS_INFO) << "Decoder implementation: " << decoder_info_.ToString();
230 if (_callback) {
231 _callback->OnDecoderImplementationName(
232 decoder_info_.implementation_name.c_str());
233 }
Danil Chapovalov355b8d22021-08-13 16:50:37 +0200234 return ok;
niklase@google.com470e71d2011-07-07 08:21:25 +0000235}
236
Johannes Kron05f84872020-01-16 14:09:33 +0100237int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, Timestamp now) {
Yves Gerey665174f2018-06-19 15:03:05 +0200238 TRACE_EVENT1("webrtc", "VCMGenericDecoder::Decode", "timestamp",
Niels Möller23775882018-08-16 10:24:12 +0200239 frame.Timestamp());
Johannes Kronb6b782d2021-03-03 14:39:44 +0100240 VCMFrameInformation frame_info;
241 frame_info.decodeStart = now;
242 frame_info.renderTimeMs = frame.RenderTimeMs();
243 frame_info.rotation = frame.rotation();
244 frame_info.timing = frame.video_timing();
245 frame_info.ntp_time_ms = frame.EncodedImage().ntp_time_ms_;
246 frame_info.packet_infos = frame.PacketInfos();
Ilya Nikolaevskiy2ebf5232019-05-13 16:13:36 +0200247
Yves Gerey665174f2018-06-19 15:03:05 +0200248 // Set correctly only for key frames. Thus, use latest key frame
249 // content type. If the corresponding key frame was lost, decode will fail
250 // and content type will be ignored.
Niels Möller8f7ce222019-03-21 15:43:58 +0100251 if (frame.FrameType() == VideoFrameType::kVideoFrameKey) {
Johannes Kronb6b782d2021-03-03 14:39:44 +0100252 frame_info.content_type = frame.contentType();
Yves Gerey665174f2018-06-19 15:03:05 +0200253 _last_keyframe_content_type = frame.contentType();
254 } else {
Johannes Kronb6b782d2021-03-03 14:39:44 +0100255 frame_info.content_type = _last_keyframe_content_type;
Yves Gerey665174f2018-06-19 15:03:05 +0200256 }
Johannes Kronb6b782d2021-03-03 14:39:44 +0100257 _callback->Map(frame.Timestamp(), frame_info);
niklase@google.com470e71d2011-07-07 08:21:25 +0000258
Yves Gerey665174f2018-06-19 15:03:05 +0200259 int32_t ret = decoder_->Decode(frame.EncodedImage(), frame.MissingFrame(),
Niels Möller7aacdd92019-03-25 09:11:40 +0100260 frame.RenderTimeMs());
Erik Språngc12f6252021-01-13 21:49:59 +0100261 VideoDecoder::DecoderInfo decoder_info = decoder_->GetDecoderInfo();
262 if (decoder_info != decoder_info_) {
Ilya Nikolaevskiy43c108b2020-05-15 12:24:29 +0200263 RTC_LOG(LS_INFO) << "Changed decoder implementation to: "
Erik Språngc12f6252021-01-13 21:49:59 +0100264 << decoder_info.ToString();
Erik Språngc80f9552021-03-12 16:36:49 +0100265 decoder_info_ = decoder_info;
Erik Språngc12f6252021-01-13 21:49:59 +0100266 _callback->OnDecoderImplementationName(
267 decoder_info.implementation_name.empty()
268 ? "unknown"
269 : decoder_info.implementation_name.c_str());
Ilya Nikolaevskiy43c108b2020-05-15 12:24:29 +0200270 }
Yves Gerey665174f2018-06-19 15:03:05 +0200271 if (ret < WEBRTC_VIDEO_CODEC_OK) {
272 RTC_LOG(LS_WARNING) << "Failed to decode frame with timestamp "
Niels Möller23775882018-08-16 10:24:12 +0200273 << frame.Timestamp() << ", error code: " << ret;
Johannes Kronfc5d2762021-04-09 16:03:22 +0200274 _callback->ClearTimestampMap();
Niels Möller9cccf852018-12-04 11:01:26 +0100275 } else if (ret == WEBRTC_VIDEO_CODEC_NO_OUTPUT) {
Johannes Kronfc5d2762021-04-09 16:03:22 +0200276 // No output.
277 _callback->ClearTimestampMap();
Yves Gerey665174f2018-06-19 15:03:05 +0200278 }
279 return ret;
guidouc3372582017-04-04 07:16:21 -0700280}
niklase@google.com470e71d2011-07-07 08:21:25 +0000281
Peter Boström187db632015-12-01 17:20:01 +0100282int32_t VCMGenericDecoder::RegisterDecodeCompleteCallback(
283 VCMDecodedFrameCallback* callback) {
284 _callback = callback;
Erik Språngc12f6252021-01-13 21:49:59 +0100285 int32_t ret = decoder_->RegisterDecodeCompleteCallback(callback);
286 if (callback && !decoder_info_.implementation_name.empty()) {
287 callback->OnDecoderImplementationName(
288 decoder_info_.implementation_name.c_str());
289 }
290 return ret;
niklase@google.com470e71d2011-07-07 08:21:25 +0000291}
292
philipel9d3ab612015-12-21 04:12:39 -0800293} // namespace webrtc