niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 1 | /* |
mikhal@webrtc.org | a2031d5 | 2012-07-31 15:53:44 +0000 | [diff] [blame] | 2 | * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved. |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
Henrik Kjellander | dca1e09 | 2017-07-01 16:42:22 +0200 | [diff] [blame^] | 11 | #include "webrtc/base/checks.h" |
| 12 | #include "webrtc/base/logging.h" |
| 13 | #include "webrtc/base/timeutils.h" |
| 14 | #include "webrtc/base/trace_event.h" |
kjellander | c8fa692 | 2017-06-30 14:02:00 -0700 | [diff] [blame] | 15 | #include "webrtc/modules/video_coding/include/video_coding.h" |
Henrik Kjellander | dca1e09 | 2017-07-01 16:42:22 +0200 | [diff] [blame^] | 16 | #include "webrtc/modules/video_coding/generic_decoder.h" |
Henrik Kjellander | 2557b86 | 2015-11-18 22:00:21 +0100 | [diff] [blame] | 17 | #include "webrtc/modules/video_coding/internal_defines.h" |
Henrik Kjellander | 98f5351 | 2015-10-28 18:17:40 +0100 | [diff] [blame] | 18 | #include "webrtc/system_wrappers/include/clock.h" |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 19 | |
| 20 | namespace webrtc { |
| 21 | |
philipel | 9d3ab61 | 2015-12-21 04:12:39 -0800 | [diff] [blame] | 22 | VCMDecodedFrameCallback::VCMDecodedFrameCallback(VCMTiming* timing, |
stefan@webrtc.org | a678a3b | 2013-01-21 07:42:11 +0000 | [diff] [blame] | 23 | Clock* clock) |
tommi | d0a71ba | 2017-03-14 04:16:20 -0700 | [diff] [blame] | 24 | : _clock(clock), |
Peter Boström | b7d9a97 | 2015-12-18 16:01:11 +0100 | [diff] [blame] | 25 | _timing(timing), |
| 26 | _timestampMap(kDecoderFrameMemoryLength), |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 27 | _lastReceivedPictureID(0) { |
| 28 | ntp_offset_ = |
| 29 | _clock->CurrentNtpInMilliseconds() - _clock->TimeInMilliseconds(); |
| 30 | } |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 31 | |
philipel | 9d3ab61 | 2015-12-21 04:12:39 -0800 | [diff] [blame] | 32 | VCMDecodedFrameCallback::~VCMDecodedFrameCallback() { |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 33 | } |
| 34 | |
stefan@webrtc.org | 06887ae | 2011-10-10 14:17:46 +0000 | [diff] [blame] | 35 | void VCMDecodedFrameCallback::SetUserReceiveCallback( |
philipel | 9d3ab61 | 2015-12-21 04:12:39 -0800 | [diff] [blame] | 36 | VCMReceiveCallback* receiveCallback) { |
tommi | d0a71ba | 2017-03-14 04:16:20 -0700 | [diff] [blame] | 37 | RTC_DCHECK(construction_thread_.CalledOnValidThread()); |
| 38 | RTC_DCHECK((!_receiveCallback && receiveCallback) || |
| 39 | (_receiveCallback && !receiveCallback)); |
philipel | 9d3ab61 | 2015-12-21 04:12:39 -0800 | [diff] [blame] | 40 | _receiveCallback = receiveCallback; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 41 | } |
| 42 | |
philipel | 9d3ab61 | 2015-12-21 04:12:39 -0800 | [diff] [blame] | 43 | VCMReceiveCallback* VCMDecodedFrameCallback::UserReceiveCallback() { |
tommi | d0a71ba | 2017-03-14 04:16:20 -0700 | [diff] [blame] | 44 | // Called on the decode thread via VCMCodecDataBase::GetDecoder. |
| 45 | // The callback must always have been set before this happens. |
| 46 | RTC_DCHECK(_receiveCallback); |
philipel | 9d3ab61 | 2015-12-21 04:12:39 -0800 | [diff] [blame] | 47 | return _receiveCallback; |
wuchengli@chromium.org | 0d94c2f | 2013-08-12 14:20:49 +0000 | [diff] [blame] | 48 | } |
| 49 | |
Miguel Casas-Sanchez | 4765070 | 2015-05-29 17:21:40 -0700 | [diff] [blame] | 50 | int32_t VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage) { |
Per | 327d8ba | 2015-11-10 14:00:27 +0100 | [diff] [blame] | 51 | return Decoded(decodedImage, -1); |
| 52 | } |
| 53 | |
| 54 | int32_t VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage, |
| 55 | int64_t decode_time_ms) { |
sakal | cc452e1 | 2017-02-09 04:53:45 -0800 | [diff] [blame] | 56 | Decoded(decodedImage, |
| 57 | decode_time_ms >= 0 ? rtc::Optional<int32_t>(decode_time_ms) |
| 58 | : rtc::Optional<int32_t>(), |
| 59 | rtc::Optional<uint8_t>()); |
| 60 | return WEBRTC_VIDEO_CODEC_OK; |
| 61 | } |
| 62 | |
| 63 | void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage, |
| 64 | rtc::Optional<int32_t> decode_time_ms, |
| 65 | rtc::Optional<uint8_t> qp) { |
tommi | d0a71ba | 2017-03-14 04:16:20 -0700 | [diff] [blame] | 66 | RTC_DCHECK(_receiveCallback) << "Callback must not be null at this point"; |
philipel | 9d3ab61 | 2015-12-21 04:12:39 -0800 | [diff] [blame] | 67 | TRACE_EVENT_INSTANT1("webrtc", "VCMDecodedFrameCallback::Decoded", |
| 68 | "timestamp", decodedImage.timestamp()); |
| 69 | // TODO(holmer): We should improve this so that we can handle multiple |
| 70 | // callbacks from one call to Decode(). |
guidou | c337258 | 2017-04-04 07:16:21 -0700 | [diff] [blame] | 71 | VCMFrameInformation* frameInfo; |
| 72 | { |
| 73 | rtc::CritScope cs(&lock_); |
| 74 | frameInfo = _timestampMap.Pop(decodedImage.timestamp()); |
| 75 | } |
philipel | 9d3ab61 | 2015-12-21 04:12:39 -0800 | [diff] [blame] | 76 | |
| 77 | if (frameInfo == NULL) { |
| 78 | LOG(LS_WARNING) << "Too many frames backed up in the decoder, dropping " |
| 79 | "this one."; |
sakal | cc452e1 | 2017-02-09 04:53:45 -0800 | [diff] [blame] | 80 | return; |
philipel | 9d3ab61 | 2015-12-21 04:12:39 -0800 | [diff] [blame] | 81 | } |
| 82 | |
| 83 | const int64_t now_ms = _clock->TimeInMilliseconds(); |
sakal | cc452e1 | 2017-02-09 04:53:45 -0800 | [diff] [blame] | 84 | if (!decode_time_ms) { |
philipel | 9d3ab61 | 2015-12-21 04:12:39 -0800 | [diff] [blame] | 85 | decode_time_ms = |
sakal | cc452e1 | 2017-02-09 04:53:45 -0800 | [diff] [blame] | 86 | rtc::Optional<int32_t>(now_ms - frameInfo->decodeStartTimeMs); |
philipel | 9d3ab61 | 2015-12-21 04:12:39 -0800 | [diff] [blame] | 87 | } |
sakal | cc452e1 | 2017-02-09 04:53:45 -0800 | [diff] [blame] | 88 | _timing->StopDecodeTimer(decodedImage.timestamp(), *decode_time_ms, now_ms, |
philipel | 9d3ab61 | 2015-12-21 04:12:39 -0800 | [diff] [blame] | 89 | frameInfo->renderTimeMs); |
| 90 | |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 91 | // Report timing information. |
| 92 | if (frameInfo->timing.is_timing_frame) { |
| 93 | // Convert remote timestamps to local time from ntp timestamps. |
| 94 | frameInfo->timing.encode_start_ms -= ntp_offset_; |
| 95 | frameInfo->timing.encode_finish_ms -= ntp_offset_; |
| 96 | frameInfo->timing.packetization_finish_ms -= ntp_offset_; |
| 97 | frameInfo->timing.pacer_exit_ms -= ntp_offset_; |
| 98 | frameInfo->timing.network_timestamp_ms -= ntp_offset_; |
| 99 | frameInfo->timing.network2_timestamp_ms -= ntp_offset_; |
| 100 | // TODO(ilnik): Report timing information here. |
| 101 | // Capture time: decodedImage.ntp_time_ms() - ntp_offset |
| 102 | // Encode start: frameInfo->timing.encode_start_ms |
| 103 | // Encode finish: frameInfo->timing.encode_finish_ms |
| 104 | // Packetization done: frameInfo->timing.packetization_finish_ms |
| 105 | // Pacer exit: frameInfo->timing.pacer_exit_ms |
| 106 | // Network timestamp: frameInfo->timing.network_timestamp_ms |
| 107 | // Network2 timestamp: frameInfo->timing.network2_timestamp_ms |
| 108 | // Receive start: frameInfo->timing.receive_start_ms |
| 109 | // Receive finish: frameInfo->timing.receive_finish_ms |
| 110 | // Decode start: frameInfo->decodeStartTimeMs |
| 111 | // Decode finish: now_ms |
| 112 | // Render time: frameInfo->renderTimeMs |
| 113 | } |
| 114 | |
nisse | 1c0dea8 | 2017-01-30 02:43:18 -0800 | [diff] [blame] | 115 | decodedImage.set_timestamp_us( |
| 116 | frameInfo->renderTimeMs * rtc::kNumMicrosecsPerMillisec); |
sakal | 55d932b | 2016-09-30 06:19:08 -0700 | [diff] [blame] | 117 | decodedImage.set_rotation(frameInfo->rotation); |
ilnik | 00d802b | 2017-04-11 10:34:31 -0700 | [diff] [blame] | 118 | _receiveCallback->FrameToRender(decodedImage, qp, frameInfo->content_type); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 119 | } |
| 120 | |
philipel | 9d3ab61 | 2015-12-21 04:12:39 -0800 | [diff] [blame] | 121 | int32_t VCMDecodedFrameCallback::ReceivedDecodedReferenceFrame( |
| 122 | const uint64_t pictureId) { |
tommi | d0a71ba | 2017-03-14 04:16:20 -0700 | [diff] [blame] | 123 | return _receiveCallback->ReceivedDecodedReferenceFrame(pictureId); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 124 | } |
| 125 | |
philipel | 9d3ab61 | 2015-12-21 04:12:39 -0800 | [diff] [blame] | 126 | int32_t VCMDecodedFrameCallback::ReceivedDecodedFrame( |
| 127 | const uint64_t pictureId) { |
| 128 | _lastReceivedPictureID = pictureId; |
| 129 | return 0; |
| 130 | } |
| 131 | |
| 132 | uint64_t VCMDecodedFrameCallback::LastReceivedPictureID() const { |
| 133 | return _lastReceivedPictureID; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 134 | } |
| 135 | |
Peter Boström | b7d9a97 | 2015-12-18 16:01:11 +0100 | [diff] [blame] | 136 | void VCMDecodedFrameCallback::OnDecoderImplementationName( |
| 137 | const char* implementation_name) { |
tommi | d0a71ba | 2017-03-14 04:16:20 -0700 | [diff] [blame] | 138 | _receiveCallback->OnDecoderImplementationName(implementation_name); |
Peter Boström | b7d9a97 | 2015-12-18 16:01:11 +0100 | [diff] [blame] | 139 | } |
| 140 | |
pbos | 1968d3f | 2015-09-28 08:52:18 -0700 | [diff] [blame] | 141 | void VCMDecodedFrameCallback::Map(uint32_t timestamp, |
| 142 | VCMFrameInformation* frameInfo) { |
guidou | c337258 | 2017-04-04 07:16:21 -0700 | [diff] [blame] | 143 | rtc::CritScope cs(&lock_); |
pbos | 1968d3f | 2015-09-28 08:52:18 -0700 | [diff] [blame] | 144 | _timestampMap.Add(timestamp, frameInfo); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 145 | } |
| 146 | |
philipel | 9d3ab61 | 2015-12-21 04:12:39 -0800 | [diff] [blame] | 147 | int32_t VCMDecodedFrameCallback::Pop(uint32_t timestamp) { |
guidou | c337258 | 2017-04-04 07:16:21 -0700 | [diff] [blame] | 148 | rtc::CritScope cs(&lock_); |
| 149 | if (_timestampMap.Pop(timestamp) == NULL) { |
| 150 | return VCM_GENERAL_ERROR; |
| 151 | } |
| 152 | return VCM_OK; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 153 | } |
| 154 | |
Peter Boström | 187db63 | 2015-12-01 17:20:01 +0100 | [diff] [blame] | 155 | VCMGenericDecoder::VCMGenericDecoder(VideoDecoder* decoder, bool isExternal) |
| 156 | : _callback(NULL), |
| 157 | _frameInfos(), |
| 158 | _nextFrameInfoIdx(0), |
guidou | c337258 | 2017-04-04 07:16:21 -0700 | [diff] [blame] | 159 | _decoder(decoder), |
Peter Boström | 187db63 | 2015-12-01 17:20:01 +0100 | [diff] [blame] | 160 | _codecType(kVideoCodecUnknown), |
guidou | c337258 | 2017-04-04 07:16:21 -0700 | [diff] [blame] | 161 | _isExternal(isExternal), |
ilnik | 00d802b | 2017-04-11 10:34:31 -0700 | [diff] [blame] | 162 | _keyFrameDecoded(false), |
| 163 | _last_keyframe_content_type(VideoContentType::UNSPECIFIED) {} |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 164 | |
guidou | c337258 | 2017-04-04 07:16:21 -0700 | [diff] [blame] | 165 | VCMGenericDecoder::~VCMGenericDecoder() {} |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 166 | |
pbos@webrtc.org | 7b859cc | 2013-04-02 15:54:38 +0000 | [diff] [blame] | 167 | int32_t VCMGenericDecoder::InitDecode(const VideoCodec* settings, |
philipel | 9d3ab61 | 2015-12-21 04:12:39 -0800 | [diff] [blame] | 168 | int32_t numberOfCores) { |
| 169 | TRACE_EVENT0("webrtc", "VCMGenericDecoder::InitDecode"); |
| 170 | _codecType = settings->codecType; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 171 | |
guidou | c337258 | 2017-04-04 07:16:21 -0700 | [diff] [blame] | 172 | return _decoder->InitDecode(settings, numberOfCores); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 173 | } |
| 174 | |
pbos | d9eec76 | 2015-11-17 06:03:43 -0800 | [diff] [blame] | 175 | int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, int64_t nowMs) { |
guidou | c337258 | 2017-04-04 07:16:21 -0700 | [diff] [blame] | 176 | TRACE_EVENT1("webrtc", "VCMGenericDecoder::Decode", "timestamp", |
| 177 | frame.EncodedImage()._timeStamp); |
| 178 | _frameInfos[_nextFrameInfoIdx].decodeStartTimeMs = nowMs; |
| 179 | _frameInfos[_nextFrameInfoIdx].renderTimeMs = frame.RenderTimeMs(); |
| 180 | _frameInfos[_nextFrameInfoIdx].rotation = frame.rotation(); |
ilnik | 04f4d12 | 2017-06-19 07:18:55 -0700 | [diff] [blame] | 181 | _frameInfos[_nextFrameInfoIdx].timing = frame.video_timing(); |
ilnik | 00d802b | 2017-04-11 10:34:31 -0700 | [diff] [blame] | 182 | // Set correctly only for key frames. Thus, use latest key frame |
| 183 | // content type. If the corresponding key frame was lost, decode will fail |
| 184 | // and content type will be ignored. |
| 185 | if (frame.FrameType() == kVideoFrameKey) { |
| 186 | _frameInfos[_nextFrameInfoIdx].content_type = frame.contentType(); |
| 187 | _last_keyframe_content_type = frame.contentType(); |
| 188 | } else { |
| 189 | _frameInfos[_nextFrameInfoIdx].content_type = _last_keyframe_content_type; |
| 190 | } |
guidou | c337258 | 2017-04-04 07:16:21 -0700 | [diff] [blame] | 191 | _callback->Map(frame.TimeStamp(), &_frameInfos[_nextFrameInfoIdx]); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 192 | |
guidou | c337258 | 2017-04-04 07:16:21 -0700 | [diff] [blame] | 193 | _nextFrameInfoIdx = (_nextFrameInfoIdx + 1) % kDecoderFrameMemoryLength; |
| 194 | const RTPFragmentationHeader dummy_header; |
| 195 | int32_t ret = _decoder->Decode(frame.EncodedImage(), frame.MissingFrame(), |
| 196 | &dummy_header, |
| 197 | frame.CodecSpecific(), frame.RenderTimeMs()); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 198 | |
guidou | c337258 | 2017-04-04 07:16:21 -0700 | [diff] [blame] | 199 | _callback->OnDecoderImplementationName(_decoder->ImplementationName()); |
philipel | 9d3ab61 | 2015-12-21 04:12:39 -0800 | [diff] [blame] | 200 | if (ret < WEBRTC_VIDEO_CODEC_OK) { |
guidou | c337258 | 2017-04-04 07:16:21 -0700 | [diff] [blame] | 201 | LOG(LS_WARNING) << "Failed to decode frame with timestamp " |
| 202 | << frame.TimeStamp() << ", error code: " << ret; |
| 203 | _callback->Pop(frame.TimeStamp()); |
| 204 | return ret; |
| 205 | } else if (ret == WEBRTC_VIDEO_CODEC_NO_OUTPUT || |
| 206 | ret == WEBRTC_VIDEO_CODEC_REQUEST_SLI) { |
| 207 | // No output |
| 208 | _callback->Pop(frame.TimeStamp()); |
stefan@webrtc.org | 06887ae | 2011-10-10 14:17:46 +0000 | [diff] [blame] | 209 | } |
guidou | c337258 | 2017-04-04 07:16:21 -0700 | [diff] [blame] | 210 | return ret; |
| 211 | } |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 212 | |
guidou | c337258 | 2017-04-04 07:16:21 -0700 | [diff] [blame] | 213 | int32_t VCMGenericDecoder::Release() { |
| 214 | return _decoder->Release(); |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 215 | } |
| 216 | |
Peter Boström | 187db63 | 2015-12-01 17:20:01 +0100 | [diff] [blame] | 217 | int32_t VCMGenericDecoder::RegisterDecodeCompleteCallback( |
| 218 | VCMDecodedFrameCallback* callback) { |
| 219 | _callback = callback; |
guidou | c337258 | 2017-04-04 07:16:21 -0700 | [diff] [blame] | 220 | return _decoder->RegisterDecodeCompleteCallback(callback); |
| 221 | } |
| 222 | |
| 223 | bool VCMGenericDecoder::External() const { |
| 224 | return _isExternal; |
niklase@google.com | 470e71d | 2011-07-07 08:21:25 +0000 | [diff] [blame] | 225 | } |
| 226 | |
perkj | 796cfaf | 2015-12-10 09:27:38 -0800 | [diff] [blame] | 227 | bool VCMGenericDecoder::PrefersLateDecoding() const { |
guidou | c337258 | 2017-04-04 07:16:21 -0700 | [diff] [blame] | 228 | return _decoder->PrefersLateDecoding(); |
perkj | 796cfaf | 2015-12-10 09:27:38 -0800 | [diff] [blame] | 229 | } |
| 230 | |
philipel | 9d3ab61 | 2015-12-21 04:12:39 -0800 | [diff] [blame] | 231 | } // namespace webrtc |