Deliver video frames on Android, on the decode thread.
VideoCoding
* Adding a method for polling for frames on Android only until the capture implementation takes care of this (longer term plan).
CodecDatabase
* Add an accessor for the current decoder
* Use std::unique_ptr<> for ownership.
* Remove "Release()" and "ReleaseDecoder()". Instead just delete.
* Remove |friend| relationship between CodecDatabase and VCMGenericDecoder.
VCMDecodedFrameCallback
* DCHECKs for thread correctness.
* Remove |lock_| now that a threading model has been established and verified.
VCMGenericDecoder
* All methods now have thread checks.
* Variable access associated with thread checkers.
VideoReceiver
* Added two notification methods, DecoderThreadStarting() and DecoderThreadStopped()
* Allows us to establish a period when the decoder thread is not running and it is safe to modify variables such as callbacks, that are only read when the decoder thread is running.
* Allows us to DCHECK thread guarantees.
* Allows synchronizing callbacks from the module process thread and have them only active while the decoder thread is running.
* The above, allows us to establish two modes for the thread, single-threaded-mutable and multi-threaded-const.
* Using that knowledge, we can remove |receive_crit_| as well as locking for a number of member variables.
MediaCodecVideoDecoder
* Removed frame polling code from this class, since this is now done from the root thread function in VideoReceiveStream.
VideoReceiveStream
* On Android: Polls for decoded frames every 10ms (same interval as previously in MediaCodecVideoDecoder)
* [Un]Registers the |video_receiver_| with the module thread only around the time the decoder thread is started/stopped.
* Notifies the receiver of start/stop events of the decoder thread.
* Changed the decoder thread to use the new PlatformThread callback type.
BUG=webrtc:7361, 695438
Review-Url: https://codereview.webrtc.org/2764573002
Cr-Commit-Position: refs/heads/master@{#17527}
diff --git a/webrtc/modules/video_coding/codec_database.cc b/webrtc/modules/video_coding/codec_database.cc
index 4d5e202..29899fc 100644
--- a/webrtc/modules/video_coding/codec_database.cc
+++ b/webrtc/modules/video_coding/codec_database.cc
@@ -71,6 +71,31 @@
return h264_settings;
}
+// Create an internal Decoder given a codec type
+static std::unique_ptr<VCMGenericDecoder> CreateDecoder(VideoCodecType type) {
+ switch (type) {
+ case kVideoCodecVP8:
+ return std::unique_ptr<VCMGenericDecoder>(
+ new VCMGenericDecoder(VP8Decoder::Create()));
+ case kVideoCodecVP9:
+ return std::unique_ptr<VCMGenericDecoder>(
+ new VCMGenericDecoder(VP9Decoder::Create()));
+ case kVideoCodecI420:
+ return std::unique_ptr<VCMGenericDecoder>(
+ new VCMGenericDecoder(new I420Decoder()));
+ case kVideoCodecH264:
+ if (H264Decoder::IsSupported()) {
+ return std::unique_ptr<VCMGenericDecoder>(
+ new VCMGenericDecoder(H264Decoder::Create()));
+ }
+ break;
+ default:
+ break;
+ }
+ LOG(LS_WARNING) << "No internal decoder of this type exists.";
+ return std::unique_ptr<VCMGenericDecoder>();
+}
+
VCMDecoderMapItem::VCMDecoderMapItem(VideoCodec* settings,
int number_of_cores,
bool require_key_frame)
@@ -98,13 +123,12 @@
external_encoder_(nullptr),
internal_source_(false),
encoded_frame_callback_(encoded_frame_callback),
- ptr_decoder_(nullptr),
dec_map_(),
dec_external_map_() {}
VCMCodecDataBase::~VCMCodecDataBase() {
DeleteEncoder();
- ReleaseDecoder(ptr_decoder_);
+ ptr_decoder_.reset();
for (auto& kv : dec_map_)
delete kv.second;
for (auto& kv : dec_external_map_)
@@ -391,11 +415,10 @@
// We can't use payload_type to check if the decoder is currently in use,
// because payload type may be out of date (e.g. before we decode the first
// frame after RegisterReceiveCodec)
- if (ptr_decoder_ != nullptr &&
- ptr_decoder_->_decoder == (*it).second->external_decoder_instance) {
+ if (ptr_decoder_ &&
+ ptr_decoder_->IsSameDecoder((*it).second->external_decoder_instance)) {
// Release it if it was registered and in use.
- ReleaseDecoder(ptr_decoder_);
- ptr_decoder_ = nullptr;
+ ptr_decoder_.reset();
}
DeregisterReceiveCodec(payload_type);
delete it->second;
@@ -455,12 +478,11 @@
RTC_DCHECK(decoded_frame_callback->UserReceiveCallback());
uint8_t payload_type = frame.PayloadType();
if (payload_type == receive_codec_.plType || payload_type == 0) {
- return ptr_decoder_;
+ return ptr_decoder_.get();
}
// Check for exisitng decoder, if exists - delete.
if (ptr_decoder_) {
- ReleaseDecoder(ptr_decoder_);
- ptr_decoder_ = nullptr;
+ ptr_decoder_.reset();
memset(&receive_codec_, 0, sizeof(VideoCodec));
}
ptr_decoder_ = CreateAndInitDecoder(frame, &receive_codec_);
@@ -471,36 +493,26 @@
callback->OnIncomingPayloadType(receive_codec_.plType);
if (ptr_decoder_->RegisterDecodeCompleteCallback(decoded_frame_callback) <
0) {
- ReleaseDecoder(ptr_decoder_);
- ptr_decoder_ = nullptr;
+ ptr_decoder_.reset();
memset(&receive_codec_, 0, sizeof(VideoCodec));
return nullptr;
}
- return ptr_decoder_;
+ return ptr_decoder_.get();
}
-void VCMCodecDataBase::ReleaseDecoder(VCMGenericDecoder* decoder) const {
- if (decoder) {
- RTC_DCHECK(decoder->_decoder);
- decoder->Release();
- if (!decoder->External()) {
- delete decoder->_decoder;
- }
- delete decoder;
- }
+VCMGenericDecoder* VCMCodecDataBase::GetCurrentDecoder() {
+ return ptr_decoder_.get();
}
bool VCMCodecDataBase::PrefersLateDecoding() const {
- if (!ptr_decoder_)
- return true;
- return ptr_decoder_->PrefersLateDecoding();
+ return ptr_decoder_ ? ptr_decoder_->PrefersLateDecoding() : true;
}
bool VCMCodecDataBase::MatchesCurrentResolution(int width, int height) const {
return send_codec_.width == width && send_codec_.height == height;
}
-VCMGenericDecoder* VCMCodecDataBase::CreateAndInitDecoder(
+std::unique_ptr<VCMGenericDecoder> VCMCodecDataBase::CreateAndInitDecoder(
const VCMEncodedFrame& frame,
VideoCodec* new_codec) const {
uint8_t payload_type = frame.PayloadType();
@@ -513,13 +525,13 @@
<< static_cast<int>(payload_type);
return nullptr;
}
- VCMGenericDecoder* ptr_decoder = nullptr;
+ std::unique_ptr<VCMGenericDecoder> ptr_decoder;
const VCMExtDecoderMapItem* external_dec_item =
FindExternalDecoderItem(payload_type);
if (external_dec_item) {
// External codec.
- ptr_decoder = new VCMGenericDecoder(
- external_dec_item->external_decoder_instance, true);
+ ptr_decoder.reset(new VCMGenericDecoder(
+ external_dec_item->external_decoder_instance, true));
} else {
// Create decoder.
ptr_decoder = CreateDecoder(decoder_item->settings->codecType);
@@ -538,7 +550,6 @@
}
if (ptr_decoder->InitDecode(decoder_item->settings.get(),
decoder_item->number_of_cores) < 0) {
- ReleaseDecoder(ptr_decoder);
return nullptr;
}
memcpy(new_codec, decoder_item->settings.get(), sizeof(VideoCodec));
@@ -552,26 +563,6 @@
ptr_encoder_.reset();
}
-VCMGenericDecoder* VCMCodecDataBase::CreateDecoder(VideoCodecType type) const {
- switch (type) {
- case kVideoCodecVP8:
- return new VCMGenericDecoder(VP8Decoder::Create());
- case kVideoCodecVP9:
- return new VCMGenericDecoder(VP9Decoder::Create());
- case kVideoCodecI420:
- return new VCMGenericDecoder(new I420Decoder());
- case kVideoCodecH264:
- if (H264Decoder::IsSupported()) {
- return new VCMGenericDecoder(H264Decoder::Create());
- }
- break;
- default:
- break;
- }
- LOG(LS_WARNING) << "No internal decoder of this type exists.";
- return nullptr;
-}
-
const VCMDecoderMapItem* VCMCodecDataBase::FindDecoderItem(
uint8_t payload_type) const {
DecoderMap::const_iterator it = dec_map_.find(payload_type);
diff --git a/webrtc/modules/video_coding/codec_database.h b/webrtc/modules/video_coding/codec_database.h
index 1317e22..5f8656d 100644
--- a/webrtc/modules/video_coding/codec_database.h
+++ b/webrtc/modules/video_coding/codec_database.h
@@ -107,9 +107,9 @@
const VCMEncodedFrame& frame,
VCMDecodedFrameCallback* decoded_frame_callback);
- // Deletes the memory of the decoder instance |decoder|. Used to delete
- // deep copies returned by CreateDecoderCopy().
- void ReleaseDecoder(VCMGenericDecoder* decoder) const;
+ // Returns the current decoder (i.e. the same value as was last returned from
+ // GetDecoder();
+ VCMGenericDecoder* GetCurrentDecoder();
// Returns true if the currently active decoder prefer to decode frames late.
// That means that frames must be decoded near the render times stamp.
@@ -121,8 +121,9 @@
typedef std::map<uint8_t, VCMDecoderMapItem*> DecoderMap;
typedef std::map<uint8_t, VCMExtDecoderMapItem*> ExternalDecoderMap;
- VCMGenericDecoder* CreateAndInitDecoder(const VCMEncodedFrame& frame,
- VideoCodec* new_codec) const;
+ std::unique_ptr<VCMGenericDecoder> CreateAndInitDecoder(
+ const VCMEncodedFrame& frame,
+ VideoCodec* new_codec) const;
// Determines whether a new codec has to be created or not.
// Checks every setting apart from maxFramerate and startBitrate.
@@ -130,9 +131,6 @@
void DeleteEncoder();
- // Create an internal Decoder given a codec type
- VCMGenericDecoder* CreateDecoder(VideoCodecType type) const;
-
const VCMDecoderMapItem* FindDecoderItem(uint8_t payload_type) const;
const VCMExtDecoderMapItem* FindExternalDecoderItem(
@@ -149,7 +147,7 @@
bool internal_source_;
VCMEncodedFrameCallback* const encoded_frame_callback_;
std::unique_ptr<VCMGenericEncoder> ptr_encoder_;
- VCMGenericDecoder* ptr_decoder_;
+ std::unique_ptr<VCMGenericDecoder> ptr_decoder_;
DecoderMap dec_map_;
ExternalDecoderMap dec_external_map_;
}; // VCMCodecDataBase
diff --git a/webrtc/modules/video_coding/generic_decoder.cc b/webrtc/modules/video_coding/generic_decoder.cc
index 2121ab6..6530a07 100644
--- a/webrtc/modules/video_coding/generic_decoder.cc
+++ b/webrtc/modules/video_coding/generic_decoder.cc
@@ -8,11 +8,12 @@
* be found in the AUTHORS file in the root of the source tree.
*/
+#include "webrtc/modules/video_coding/generic_decoder.h"
+
#include "webrtc/base/checks.h"
#include "webrtc/base/logging.h"
#include "webrtc/base/trace_event.h"
#include "webrtc/modules/video_coding/include/video_coding.h"
-#include "webrtc/modules/video_coding/generic_decoder.h"
#include "webrtc/modules/video_coding/internal_defines.h"
#include "webrtc/system_wrappers/include/clock.h"
@@ -23,9 +24,12 @@
: _clock(clock),
_timing(timing),
_timestampMap(kDecoderFrameMemoryLength),
- _lastReceivedPictureID(0) {}
+ _lastReceivedPictureID(0) {
+ decoder_thread_.DetachFromThread();
+}
VCMDecodedFrameCallback::~VCMDecodedFrameCallback() {
+ RTC_DCHECK(construction_thread_.CalledOnValidThread());
}
void VCMDecodedFrameCallback::SetUserReceiveCallback(
@@ -37,6 +41,7 @@
}
VCMReceiveCallback* VCMDecodedFrameCallback::UserReceiveCallback() {
+ RTC_DCHECK_RUN_ON(&decoder_thread_);
// Called on the decode thread via VCMCodecDataBase::GetDecoder.
// The callback must always have been set before this happens.
RTC_DCHECK(_receiveCallback);
@@ -59,16 +64,14 @@
void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage,
rtc::Optional<int32_t> decode_time_ms,
rtc::Optional<uint8_t> qp) {
+ RTC_DCHECK_RUN_ON(&decoder_thread_);
RTC_DCHECK(_receiveCallback) << "Callback must not be null at this point";
+
TRACE_EVENT_INSTANT1("webrtc", "VCMDecodedFrameCallback::Decoded",
"timestamp", decodedImage.timestamp());
// TODO(holmer): We should improve this so that we can handle multiple
// callbacks from one call to Decode().
- VCMFrameInformation* frameInfo;
- {
- rtc::CritScope cs(&lock_);
- frameInfo = _timestampMap.Pop(decodedImage.timestamp());
- }
+ VCMFrameInformation* frameInfo = _timestampMap.Pop(decodedImage.timestamp());
if (frameInfo == NULL) {
LOG(LS_WARNING) << "Too many frames backed up in the decoder, dropping "
@@ -92,101 +95,115 @@
int32_t VCMDecodedFrameCallback::ReceivedDecodedReferenceFrame(
const uint64_t pictureId) {
+ RTC_DCHECK_RUN_ON(&decoder_thread_);
return _receiveCallback->ReceivedDecodedReferenceFrame(pictureId);
}
int32_t VCMDecodedFrameCallback::ReceivedDecodedFrame(
const uint64_t pictureId) {
+ RTC_DCHECK_RUN_ON(&decoder_thread_);
_lastReceivedPictureID = pictureId;
return 0;
}
uint64_t VCMDecodedFrameCallback::LastReceivedPictureID() const {
+ RTC_DCHECK_RUN_ON(&decoder_thread_);
return _lastReceivedPictureID;
}
void VCMDecodedFrameCallback::OnDecoderImplementationName(
const char* implementation_name) {
+ RTC_DCHECK_RUN_ON(&decoder_thread_);
_receiveCallback->OnDecoderImplementationName(implementation_name);
}
void VCMDecodedFrameCallback::Map(uint32_t timestamp,
VCMFrameInformation* frameInfo) {
- rtc::CritScope cs(&lock_);
+ RTC_DCHECK_RUN_ON(&decoder_thread_);
_timestampMap.Add(timestamp, frameInfo);
}
int32_t VCMDecodedFrameCallback::Pop(uint32_t timestamp) {
- rtc::CritScope cs(&lock_);
- if (_timestampMap.Pop(timestamp) == NULL) {
- return VCM_GENERAL_ERROR;
- }
- return VCM_OK;
+ RTC_DCHECK_RUN_ON(&decoder_thread_);
+ return _timestampMap.Pop(timestamp) == nullptr ? VCM_GENERAL_ERROR : VCM_OK;
}
VCMGenericDecoder::VCMGenericDecoder(VideoDecoder* decoder, bool isExternal)
: _callback(NULL),
_frameInfos(),
_nextFrameInfoIdx(0),
- _decoder(decoder),
+ decoder_(decoder),
_codecType(kVideoCodecUnknown),
- _isExternal(isExternal),
- _keyFrameDecoded(false) {}
+ _isExternal(isExternal) {}
-VCMGenericDecoder::~VCMGenericDecoder() {}
+VCMGenericDecoder::~VCMGenericDecoder() {
+ decoder_->Release();
+ if (_isExternal)
+ decoder_.release();
+
+ RTC_DCHECK(_isExternal || !decoder_);
+}
int32_t VCMGenericDecoder::InitDecode(const VideoCodec* settings,
int32_t numberOfCores) {
+ RTC_DCHECK_RUN_ON(&decoder_thread_);
TRACE_EVENT0("webrtc", "VCMGenericDecoder::InitDecode");
_codecType = settings->codecType;
- return _decoder->InitDecode(settings, numberOfCores);
+ return decoder_->InitDecode(settings, numberOfCores);
}
int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, int64_t nowMs) {
- TRACE_EVENT1("webrtc", "VCMGenericDecoder::Decode", "timestamp",
- frame.EncodedImage()._timeStamp);
- _frameInfos[_nextFrameInfoIdx].decodeStartTimeMs = nowMs;
- _frameInfos[_nextFrameInfoIdx].renderTimeMs = frame.RenderTimeMs();
- _frameInfos[_nextFrameInfoIdx].rotation = frame.rotation();
- _callback->Map(frame.TimeStamp(), &_frameInfos[_nextFrameInfoIdx]);
+ RTC_DCHECK_RUN_ON(&decoder_thread_);
+ TRACE_EVENT2("webrtc", "VCMGenericDecoder::Decode", "timestamp",
+ frame.EncodedImage()._timeStamp, "decoder",
+ decoder_->ImplementationName());
+ _frameInfos[_nextFrameInfoIdx].decodeStartTimeMs = nowMs;
+ _frameInfos[_nextFrameInfoIdx].renderTimeMs = frame.RenderTimeMs();
+ _frameInfos[_nextFrameInfoIdx].rotation = frame.rotation();
+ _callback->Map(frame.TimeStamp(), &_frameInfos[_nextFrameInfoIdx]);
- _nextFrameInfoIdx = (_nextFrameInfoIdx + 1) % kDecoderFrameMemoryLength;
- const RTPFragmentationHeader dummy_header;
- int32_t ret = _decoder->Decode(frame.EncodedImage(), frame.MissingFrame(),
- &dummy_header,
- frame.CodecSpecific(), frame.RenderTimeMs());
+ _nextFrameInfoIdx = (_nextFrameInfoIdx + 1) % kDecoderFrameMemoryLength;
+ const RTPFragmentationHeader dummy_header;
+ int32_t ret = decoder_->Decode(frame.EncodedImage(), frame.MissingFrame(),
+ &dummy_header, frame.CodecSpecific(),
+ frame.RenderTimeMs());
- _callback->OnDecoderImplementationName(_decoder->ImplementationName());
+ // TODO(tommi): Necessary every time?
+ // Maybe this should be the first thing the function does, and only the first
+ // time around?
+ _callback->OnDecoderImplementationName(decoder_->ImplementationName());
+
+ if (ret != WEBRTC_VIDEO_CODEC_OK) {
if (ret < WEBRTC_VIDEO_CODEC_OK) {
- LOG(LS_WARNING) << "Failed to decode frame with timestamp "
- << frame.TimeStamp() << ", error code: " << ret;
- _callback->Pop(frame.TimeStamp());
- return ret;
- } else if (ret == WEBRTC_VIDEO_CODEC_NO_OUTPUT ||
- ret == WEBRTC_VIDEO_CODEC_REQUEST_SLI) {
- // No output
- _callback->Pop(frame.TimeStamp());
+ LOG(LS_WARNING) << "Failed to decode frame with timestamp "
+ << frame.TimeStamp() << ", error code: " << ret;
}
- return ret;
-}
+ // We pop the frame for all non-'OK', failure or success codes such as
+ // WEBRTC_VIDEO_CODEC_NO_OUTPUT and WEBRTC_VIDEO_CODEC_REQUEST_SLI.
+ _callback->Pop(frame.TimeStamp());
+ }
-int32_t VCMGenericDecoder::Release() {
- return _decoder->Release();
+ return ret;
}
int32_t VCMGenericDecoder::RegisterDecodeCompleteCallback(
VCMDecodedFrameCallback* callback) {
+ RTC_DCHECK_RUN_ON(&decoder_thread_);
_callback = callback;
- return _decoder->RegisterDecodeCompleteCallback(callback);
-}
-
-bool VCMGenericDecoder::External() const {
- return _isExternal;
+ return decoder_->RegisterDecodeCompleteCallback(callback);
}
bool VCMGenericDecoder::PrefersLateDecoding() const {
- return _decoder->PrefersLateDecoding();
+ RTC_DCHECK_RUN_ON(&decoder_thread_);
+ return decoder_->PrefersLateDecoding();
}
+#if defined(WEBRTC_ANDROID)
+void VCMGenericDecoder::PollDecodedFrames() {
+ RTC_DCHECK_RUN_ON(&decoder_thread_);
+ decoder_->PollDecodedFrames();
+}
+#endif
+
} // namespace webrtc
diff --git a/webrtc/modules/video_coding/generic_decoder.h b/webrtc/modules/video_coding/generic_decoder.h
index 891ec89..1df2060 100644
--- a/webrtc/modules/video_coding/generic_decoder.h
+++ b/webrtc/modules/video_coding/generic_decoder.h
@@ -11,7 +11,8 @@
#ifndef WEBRTC_MODULES_VIDEO_CODING_GENERIC_DECODER_H_
#define WEBRTC_MODULES_VIDEO_CODING_GENERIC_DECODER_H_
-#include "webrtc/base/criticalsection.h"
+#include <memory>
+
#include "webrtc/base/thread_checker.h"
#include "webrtc/modules/include/module_common_types.h"
#include "webrtc/modules/video_coding/encoded_frame.h"
@@ -36,6 +37,7 @@
public:
VCMDecodedFrameCallback(VCMTiming* timing, Clock* clock);
~VCMDecodedFrameCallback() override;
+
void SetUserReceiveCallback(VCMReceiveCallback* receiveCallback);
VCMReceiveCallback* UserReceiveCallback();
@@ -55,7 +57,7 @@
private:
rtc::ThreadChecker construction_thread_;
- // Protect |_timestampMap|.
+ rtc::ThreadChecker decoder_thread_;
Clock* const _clock;
// This callback must be set before the decoder thread starts running
// and must only be unset when external threads (e.g decoder thread)
@@ -63,15 +65,12 @@
// while there are more than one threads involved, it must be set
// from the same thread, and therfore a lock is not required to access it.
VCMReceiveCallback* _receiveCallback = nullptr;
- VCMTiming* _timing;
- rtc::CriticalSection lock_;
- VCMTimestampMap _timestampMap GUARDED_BY(lock_);
- uint64_t _lastReceivedPictureID;
+ VCMTiming* _timing ACCESS_ON(decoder_thread_);
+ VCMTimestampMap _timestampMap ACCESS_ON(decoder_thread_);
+ uint64_t _lastReceivedPictureID ACCESS_ON(decoder_thread_);
};
class VCMGenericDecoder {
- friend class VCMCodecDataBase;
-
public:
explicit VCMGenericDecoder(VideoDecoder* decoder, bool isExternal = false);
~VCMGenericDecoder();
@@ -89,26 +88,30 @@
int32_t Decode(const VCMEncodedFrame& inputFrame, int64_t nowMs);
/**
- * Free the decoder memory
- */
- int32_t Release();
-
- /**
* Set decode callback. Deregistering while decoding is illegal.
*/
int32_t RegisterDecodeCompleteCallback(VCMDecodedFrameCallback* callback);
- bool External() const;
bool PrefersLateDecoding() const;
+#if defined(WEBRTC_ANDROID)
+ // See https://bugs.chromium.org/p/webrtc/issues/detail?id=7361
+ void PollDecodedFrames();
+#endif
+
+ bool IsSameDecoder(VideoDecoder* decoder) const {
+ return decoder_.get() == decoder;
+ }
+
private:
- VCMDecodedFrameCallback* _callback;
- VCMFrameInformation _frameInfos[kDecoderFrameMemoryLength];
- uint32_t _nextFrameInfoIdx;
- VideoDecoder* const _decoder;
- VideoCodecType _codecType;
- bool _isExternal;
- bool _keyFrameDecoded;
+ rtc::ThreadChecker decoder_thread_;
+ VCMDecodedFrameCallback* _callback ACCESS_ON(decoder_thread_);
+ VCMFrameInformation _frameInfos[kDecoderFrameMemoryLength] ACCESS_ON(
+ decoder_thread_);
+ uint32_t _nextFrameInfoIdx ACCESS_ON(decoder_thread_);
+ std::unique_ptr<VideoDecoder> decoder_;
+ VideoCodecType _codecType ACCESS_ON(decoder_thread_);
+ const bool _isExternal;
};
} // namespace webrtc
diff --git a/webrtc/modules/video_coding/video_coding_impl.h b/webrtc/modules/video_coding/video_coding_impl.h
index 5e547df..d6e722d 100644
--- a/webrtc/modules/video_coding/video_coding_impl.h
+++ b/webrtc/modules/video_coding/video_coding_impl.h
@@ -35,6 +35,7 @@
namespace webrtc {
+class ProcessThread;
class VideoBitrateAllocator;
class VideoBitrateAllocationObserver;
@@ -150,7 +151,7 @@
VCMTiming* timing,
NackSender* nack_sender = nullptr,
KeyFrameRequestSender* keyframe_request_sender = nullptr);
- ~VideoReceiver();
+ ~VideoReceiver() override;
int32_t RegisterReceiveCodec(const VideoCodec* receiveCodec,
int32_t numberOfCores,
@@ -168,8 +169,10 @@
int32_t Decode(const webrtc::VCMEncodedFrame* frame);
- // Called on the decoder thread when thread is exiting.
- void DecodingStopped();
+#if defined(WEBRTC_ANDROID)
+ // See https://bugs.chromium.org/p/webrtc/issues/detail?id=7361
+ void PollDecodedFrames();
+#endif
int32_t IncomingPacket(const uint8_t* incomingPayload,
size_t payloadLength,
@@ -195,39 +198,66 @@
int64_t TimeUntilNextProcess() override;
void Process() override;
+ void ProcessThreadAttached(ProcessThread* process_thread) override;
void TriggerDecoderShutdown();
+ void DecoderThreadStarting();
+ void DecoderThreadStopped();
protected:
- int32_t Decode(const webrtc::VCMEncodedFrame& frame)
- EXCLUSIVE_LOCKS_REQUIRED(receive_crit_);
+ int32_t Decode(const webrtc::VCMEncodedFrame& frame);
int32_t RequestKeyFrame();
private:
+ // Used for DCHECKing thread correctness.
+ // In build where DCHECKs are enabled, will return false before
+ // DecoderThreadStarting is called, then true until DecoderThreadStopped
+ // is called.
+ // In builds where DCHECKs aren't enabled, it will return true.
+ bool IsDecoderThreadRunning();
+
rtc::ThreadChecker construction_thread_;
+ rtc::ThreadChecker decoder_thread_;
+ rtc::ThreadChecker module_thread_;
Clock* const clock_;
rtc::CriticalSection process_crit_;
- rtc::CriticalSection receive_crit_;
VCMTiming* _timing;
VCMReceiver _receiver;
VCMDecodedFrameCallback _decodedFrameCallback;
- VCMFrameTypeCallback* _frameTypeCallback GUARDED_BY(process_crit_);
- VCMReceiveStatisticsCallback* _receiveStatsCallback GUARDED_BY(process_crit_);
- VCMPacketRequestCallback* _packetRequestCallback GUARDED_BY(process_crit_);
- VCMFrameBuffer _frameFromFile;
+ // These callbacks are set on the construction thread before being attached
+ // to the module thread or decoding started, so a lock is not required.
+ VCMFrameTypeCallback* _frameTypeCallback;
+ VCMReceiveStatisticsCallback* _receiveStatsCallback;
+ VCMPacketRequestCallback* _packetRequestCallback;
+
+ // Used on both the module and decoder thread.
bool _scheduleKeyRequest GUARDED_BY(process_crit_);
bool drop_frames_until_keyframe_ GUARDED_BY(process_crit_);
- size_t max_nack_list_size_ GUARDED_BY(process_crit_);
- VCMCodecDataBase _codecDataBase GUARDED_BY(receive_crit_);
- EncodedImageCallback* pre_decode_image_callback_;
+ // Modified on the construction thread while not attached to the process
+ // thread. Once attached to the process thread, its value is only read
+ // so a lock is not required.
+ size_t max_nack_list_size_;
- VCMProcessTimer _receiveStatsTimer;
- VCMProcessTimer _retransmissionTimer;
- VCMProcessTimer _keyRequestTimer;
- QpParser qp_parser_;
- ThreadUnsafeOneTimeEvent first_frame_received_;
+ // Callbacks are set before the decoder thread starts.
+ // Once the decoder thread has been started, usage of |_codecDataBase| moves
+ // over to the decoder thread.
+ VCMCodecDataBase _codecDataBase;
+ EncodedImageCallback* const pre_decode_image_callback_;
+
+ VCMProcessTimer _receiveStatsTimer ACCESS_ON(module_thread_);
+ VCMProcessTimer _retransmissionTimer ACCESS_ON(module_thread_);
+ VCMProcessTimer _keyRequestTimer ACCESS_ON(module_thread_);
+ QpParser qp_parser_ ACCESS_ON(decoder_thread_);
+ ThreadUnsafeOneTimeEvent first_frame_received_ ACCESS_ON(decoder_thread_);
+ // Modified on the construction thread. Can be read without a lock and assumed
+ // to be non-null on the module and decoder threads.
+ ProcessThread* process_thread_ = nullptr;
+ bool is_attached_to_process_thread_ ACCESS_ON(construction_thread_) = false;
+#if RTC_DCHECK_IS_ON
+ bool decoder_thread_is_running_ = false;
+#endif
};
} // namespace vcm
diff --git a/webrtc/modules/video_coding/video_receiver.cc b/webrtc/modules/video_coding/video_receiver.cc
index 84f5af5..f1503ba 100644
--- a/webrtc/modules/video_coding/video_receiver.cc
+++ b/webrtc/modules/video_coding/video_receiver.cc
@@ -9,12 +9,14 @@
*/
#include "webrtc/base/checks.h"
+#include "webrtc/base/location.h"
#include "webrtc/base/logging.h"
#include "webrtc/base/trace_event.h"
#include "webrtc/common_types.h"
#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
-#include "webrtc/modules/video_coding/include/video_codec_interface.h"
+#include "webrtc/modules/utility/include/process_thread.h"
#include "webrtc/modules/video_coding/encoded_frame.h"
+#include "webrtc/modules/video_coding/include/video_codec_interface.h"
#include "webrtc/modules/video_coding/jitter_buffer.h"
#include "webrtc/modules/video_coding/packet.h"
#include "webrtc/modules/video_coding/video_coding_impl.h"
@@ -40,7 +42,6 @@
_frameTypeCallback(nullptr),
_receiveStatsCallback(nullptr),
_packetRequestCallback(nullptr),
- _frameFromFile(),
_scheduleKeyRequest(false),
drop_frames_until_keyframe_(false),
max_nack_list_size_(0),
@@ -48,18 +49,23 @@
pre_decode_image_callback_(pre_decode_image_callback),
_receiveStatsTimer(1000, clock_),
_retransmissionTimer(10, clock_),
- _keyRequestTimer(500, clock_) {}
+ _keyRequestTimer(500, clock_) {
+ decoder_thread_.DetachFromThread();
+ module_thread_.DetachFromThread();
+}
-VideoReceiver::~VideoReceiver() {}
+VideoReceiver::~VideoReceiver() {
+ RTC_DCHECK_RUN_ON(&construction_thread_);
+}
void VideoReceiver::Process() {
+ RTC_DCHECK_RUN_ON(&module_thread_);
// Receive-side statistics
// TODO(philipel): Remove this if block when we know what to do with
// ReceiveStatisticsProxy::QualitySample.
if (_receiveStatsTimer.TimeUntilProcess() == 0) {
_receiveStatsTimer.Processed();
- rtc::CritScope cs(&process_crit_);
if (_receiveStatsCallback != nullptr) {
_receiveStatsCallback->OnReceiveRatesUpdated(0, 0);
}
@@ -68,10 +74,10 @@
// Key frame requests
if (_keyRequestTimer.TimeUntilProcess() == 0) {
_keyRequestTimer.Processed();
- bool request_key_frame = false;
- {
+ bool request_key_frame = _frameTypeCallback != nullptr;
+ if (request_key_frame) {
rtc::CritScope cs(&process_crit_);
- request_key_frame = _scheduleKeyRequest && _frameTypeCallback != nullptr;
+ request_key_frame = _scheduleKeyRequest;
}
if (request_key_frame)
RequestKeyFrame();
@@ -82,13 +88,8 @@
// disabled when NACK is off.
if (_retransmissionTimer.TimeUntilProcess() == 0) {
_retransmissionTimer.Processed();
- bool callback_registered = false;
- uint16_t length;
- {
- rtc::CritScope cs(&process_crit_);
- length = max_nack_list_size_;
- callback_registered = _packetRequestCallback != nullptr;
- }
+ bool callback_registered = _packetRequestCallback != nullptr;
+ uint16_t length = max_nack_list_size_;
if (callback_registered && length > 0) {
// Collect sequence numbers from the default receiver.
bool request_key_frame = false;
@@ -98,7 +99,6 @@
ret = RequestKeyFrame();
}
if (ret == VCM_OK && !nackList.empty()) {
- rtc::CritScope cs(&process_crit_);
if (_packetRequestCallback != nullptr) {
_packetRequestCallback->ResendPackets(&nackList[0], nackList.size());
}
@@ -107,7 +107,18 @@
}
}
+void VideoReceiver::ProcessThreadAttached(ProcessThread* process_thread) {
+ RTC_DCHECK_RUN_ON(&construction_thread_);
+ if (process_thread) {
+ is_attached_to_process_thread_ = true;
+ process_thread_ = process_thread;
+ } else {
+ is_attached_to_process_thread_ = false;
+ }
+}
+
int64_t VideoReceiver::TimeUntilNextProcess() {
+ RTC_DCHECK_RUN_ON(&module_thread_);
int64_t timeUntilNextProcess = _receiveStatsTimer.TimeUntilProcess();
if (_receiver.NackMode() != kNoNack) {
// We need a Process call more often if we are relying on
@@ -122,7 +133,7 @@
}
int32_t VideoReceiver::SetReceiveChannelParameters(int64_t rtt) {
- rtc::CritScope cs(&receive_crit_);
+ RTC_DCHECK_RUN_ON(&module_thread_);
_receiver.UpdateRtt(rtt);
return 0;
}
@@ -142,7 +153,6 @@
}
case kProtectionNackFEC: {
- rtc::CritScope cs(&receive_crit_);
RTC_DCHECK(enable);
_receiver.SetNackMode(kNack,
media_optimization::kLowRttNackMs,
@@ -165,20 +175,22 @@
// ready for rendering.
int32_t VideoReceiver::RegisterReceiveCallback(
VCMReceiveCallback* receiveCallback) {
- RTC_DCHECK(construction_thread_.CalledOnValidThread());
- // TODO(tommi): Callback may be null, but only after the decoder thread has
- // been stopped. Use the signal we now get that tells us when the decoder
- // thread isn't running, to DCHECK that the method is never called while it
- // is. Once we're confident, we can remove the lock.
- rtc::CritScope cs(&receive_crit_);
+ RTC_DCHECK_RUN_ON(&construction_thread_);
+ RTC_DCHECK(!IsDecoderThreadRunning());
+ // This value is set before the decoder thread starts and unset after
+ // the decoder thread has been stopped.
_decodedFrameCallback.SetUserReceiveCallback(receiveCallback);
return VCM_OK;
}
int32_t VideoReceiver::RegisterReceiveStatisticsCallback(
VCMReceiveStatisticsCallback* receiveStats) {
- RTC_DCHECK(construction_thread_.CalledOnValidThread());
- rtc::CritScope cs(&process_crit_);
+ RTC_DCHECK_RUN_ON(&construction_thread_);
+ RTC_DCHECK(!IsDecoderThreadRunning() && !is_attached_to_process_thread_);
+ // |_receiver| is used on both the decoder and module threads.
+ // However, since we make sure that we never do anything on the module thread
+ // when the decoder thread is not running, we don't need a lock for the
+ // |_receiver| or |_receiveStatsCallback| here.
_receiver.RegisterStatsCallback(receiveStats);
_receiveStatsCallback = receiveStats;
return VCM_OK;
@@ -187,10 +199,8 @@
// Register an externally defined decoder object.
void VideoReceiver::RegisterExternalDecoder(VideoDecoder* externalDecoder,
uint8_t payloadType) {
- RTC_DCHECK(construction_thread_.CalledOnValidThread());
- // TODO(tommi): This method must be called when the decoder thread is not
- // running. Do we need a lock in that case?
- rtc::CritScope cs(&receive_crit_);
+ RTC_DCHECK_RUN_ON(&construction_thread_);
+ RTC_DCHECK(!IsDecoderThreadRunning());
if (externalDecoder == nullptr) {
RTC_CHECK(_codecDataBase.DeregisterExternalDecoder(payloadType));
return;
@@ -201,53 +211,87 @@
// Register a frame type request callback.
int32_t VideoReceiver::RegisterFrameTypeCallback(
VCMFrameTypeCallback* frameTypeCallback) {
- rtc::CritScope cs(&process_crit_);
+ RTC_DCHECK_RUN_ON(&construction_thread_);
+ RTC_DCHECK(!IsDecoderThreadRunning() && !is_attached_to_process_thread_);
+ // This callback is used on the module thread, but since we don't get
+ // callbacks on the module thread while the decoder thread isn't running
+ // (and this function must not be called when the decoder is running),
+ // we don't need a lock here.
_frameTypeCallback = frameTypeCallback;
return VCM_OK;
}
int32_t VideoReceiver::RegisterPacketRequestCallback(
VCMPacketRequestCallback* callback) {
- rtc::CritScope cs(&process_crit_);
+ RTC_DCHECK_RUN_ON(&construction_thread_);
+ RTC_DCHECK(!IsDecoderThreadRunning() && !is_attached_to_process_thread_);
+ // This callback is used on the module thread, but since we don't get
+ // callbacks on the module thread while the decoder thread isn't running
+ // (and this function must not be called when the decoder is running),
+ // we don't need a lock here.
_packetRequestCallback = callback;
return VCM_OK;
}
void VideoReceiver::TriggerDecoderShutdown() {
- RTC_DCHECK(construction_thread_.CalledOnValidThread());
+ RTC_DCHECK_RUN_ON(&construction_thread_);
+ RTC_DCHECK(IsDecoderThreadRunning());
_receiver.TriggerDecoderShutdown();
}
+void VideoReceiver::DecoderThreadStarting() {
+ RTC_DCHECK_RUN_ON(&construction_thread_);
+ RTC_DCHECK(!IsDecoderThreadRunning());
+ if (process_thread_ && !is_attached_to_process_thread_) {
+ process_thread_->RegisterModule(this, RTC_FROM_HERE);
+ }
+#if RTC_DCHECK_IS_ON
+ decoder_thread_is_running_ = true;
+#endif
+}
+
+void VideoReceiver::DecoderThreadStopped() {
+ RTC_DCHECK_RUN_ON(&construction_thread_);
+ RTC_DCHECK(IsDecoderThreadRunning());
+ if (process_thread_ && is_attached_to_process_thread_) {
+ process_thread_->DeRegisterModule(this);
+ }
+#if RTC_DCHECK_IS_ON
+ decoder_thread_is_running_ = false;
+ decoder_thread_.DetachFromThread();
+#endif
+}
+
// Decode next frame, blocking.
// Should be called as often as possible to get the most out of the decoder.
int32_t VideoReceiver::Decode(uint16_t maxWaitTimeMs) {
- bool prefer_late_decoding = false;
- {
- // TODO(tommi): Chances are that this lock isn't required.
- rtc::CritScope cs(&receive_crit_);
- prefer_late_decoding = _codecDataBase.PrefersLateDecoding();
- }
-
- VCMEncodedFrame* frame =
- _receiver.FrameForDecoding(maxWaitTimeMs, prefer_late_decoding);
+ RTC_DCHECK_RUN_ON(&decoder_thread_);
+ VCMEncodedFrame* frame = _receiver.FrameForDecoding(
+ maxWaitTimeMs, _codecDataBase.PrefersLateDecoding());
if (!frame)
return VCM_FRAME_NOT_READY;
+ bool drop_frame = false;
{
rtc::CritScope cs(&process_crit_);
if (drop_frames_until_keyframe_) {
// Still getting delta frames, schedule another keyframe request as if
// decode failed.
if (frame->FrameType() != kVideoFrameKey) {
+ drop_frame = true;
_scheduleKeyRequest = true;
- _receiver.ReleaseFrame(frame);
- return VCM_FRAME_NOT_READY;
+ } else {
+ drop_frames_until_keyframe_ = false;
}
- drop_frames_until_keyframe_ = false;
}
}
+ if (drop_frame) {
+ _receiver.ReleaseFrame(frame);
+ return VCM_FRAME_NOT_READY;
+ }
+
if (pre_decode_image_callback_) {
EncodedImage encoded_image(frame->EncodedImage());
int qp = -1;
@@ -258,7 +302,6 @@
frame->CodecSpecific(), nullptr);
}
- rtc::CritScope cs(&receive_crit_);
// If this frame was too late, we should adjust the delay accordingly
_timing->UpdateCurrentDelay(frame->RenderTimeMs(),
clock_->TimeInMilliseconds());
@@ -278,7 +321,7 @@
// TODO(philipel): Clean up among the Decode functions as we replace
// VCMEncodedFrame with FrameObject.
int32_t VideoReceiver::Decode(const webrtc::VCMEncodedFrame* frame) {
- rtc::CritScope lock(&receive_crit_);
+ RTC_DCHECK_RUN_ON(&decoder_thread_);
if (pre_decode_image_callback_) {
EncodedImage encoded_image(frame->EncodedImage());
int qp = -1;
@@ -291,19 +334,20 @@
return Decode(*frame);
}
-void VideoReceiver::DecodingStopped() {
- // No further calls to Decode() will be made after this point.
- // TODO(tommi): Make use of this to clarify and check threading model.
-}
-
int32_t VideoReceiver::RequestKeyFrame() {
+ RTC_DCHECK_RUN_ON(&module_thread_);
+
+ // Since we deregister from the module thread when the decoder thread isn't
+ // running, we should get no calls here if decoding isn't being done.
+ RTC_DCHECK(IsDecoderThreadRunning());
+
TRACE_EVENT0("webrtc", "RequestKeyFrame");
- rtc::CritScope cs(&process_crit_);
if (_frameTypeCallback != nullptr) {
const int32_t ret = _frameTypeCallback->RequestKeyFrame();
if (ret < 0) {
return ret;
}
+ rtc::CritScope cs(&process_crit_);
_scheduleKeyRequest = false;
} else {
return VCM_MISSING_CALLBACK;
@@ -313,6 +357,7 @@
// Must be called from inside the receive side critical section.
int32_t VideoReceiver::Decode(const VCMEncodedFrame& frame) {
+ RTC_DCHECK_RUN_ON(&decoder_thread_);
TRACE_EVENT0("webrtc", "VideoReceiver::Decode");
// Change decoder if payload type has changed
VCMGenericDecoder* decoder =
@@ -324,31 +369,41 @@
int32_t ret = decoder->Decode(frame, clock_->TimeInMilliseconds());
// Check for failed decoding, run frame type request callback if needed.
- bool request_key_frame = false;
- if (ret < 0) {
- request_key_frame = true;
- }
+ bool request_key_frame = (ret < 0);
if (!frame.Complete() || frame.MissingFrame()) {
request_key_frame = true;
ret = VCM_OK;
}
+
if (request_key_frame) {
rtc::CritScope cs(&process_crit_);
- _scheduleKeyRequest = true;
+ if (!_scheduleKeyRequest) {
+ _scheduleKeyRequest = true;
+ // TODO(tommi): Consider if we could instead post a task to the module
+ // thread and call RequestKeyFrame directly. Here we call WakeUp so that
+ // TimeUntilNextProcess() gets called straight away.
+ process_thread_->WakeUp(this);
+ }
}
return ret;
}
+#if defined(WEBRTC_ANDROID)
+void VideoReceiver::PollDecodedFrames() {
+ RTC_DCHECK_RUN_ON(&decoder_thread_);
+ auto* current_decoder = _codecDataBase.GetCurrentDecoder();
+ if (current_decoder)
+ current_decoder->PollDecodedFrames();
+}
+#endif
+
// Register possible receive codecs, can be called multiple times
int32_t VideoReceiver::RegisterReceiveCodec(const VideoCodec* receiveCodec,
int32_t numberOfCores,
bool requireKeyFrame) {
- RTC_DCHECK(construction_thread_.CalledOnValidThread());
- // TODO(tommi): This method must only be called when the decoder thread
- // is not running. Do we need a lock? If not, it looks like we might not need
- // a lock at all for |_codecDataBase|.
- rtc::CritScope cs(&receive_crit_);
+ RTC_DCHECK_RUN_ON(&construction_thread_);
+ RTC_DCHECK(!IsDecoderThreadRunning());
if (receiveCodec == nullptr) {
return VCM_PARAMETER_ERROR;
}
@@ -363,6 +418,7 @@
int32_t VideoReceiver::IncomingPacket(const uint8_t* incomingPayload,
size_t payloadLength,
const WebRtcRTPHeader& rtpInfo) {
+ RTC_DCHECK_RUN_ON(&module_thread_);
if (rtpInfo.frameType == kVideoFrameKey) {
TRACE_EVENT1("webrtc", "VCM::PacketKeyFrame", "seqnum",
rtpInfo.header.sequenceNumber);
@@ -394,6 +450,7 @@
// to sync with audio. Not included in VideoCodingModule::Delay()
// Defaults to 0 ms.
int32_t VideoReceiver::SetMinimumPlayoutDelay(uint32_t minPlayoutDelayMs) {
+ RTC_DCHECK_RUN_ON(&module_thread_);
_timing->set_min_playout_delay(minPlayoutDelayMs);
return VCM_OK;
}
@@ -401,22 +458,24 @@
// The estimated delay caused by rendering, defaults to
// kDefaultRenderDelayMs = 10 ms
int32_t VideoReceiver::SetRenderDelay(uint32_t timeMS) {
+ RTC_DCHECK_RUN_ON(&construction_thread_);
+ RTC_DCHECK(!IsDecoderThreadRunning());
_timing->set_render_delay(timeMS);
return VCM_OK;
}
// Current video delay
int32_t VideoReceiver::Delay() const {
+ RTC_DCHECK_RUN_ON(&module_thread_);
return _timing->TargetVideoDelay();
}
+// Only used by VCMRobustnessTest.
int VideoReceiver::SetReceiverRobustnessMode(
VideoCodingModule::ReceiverRobustness robustnessMode,
VCMDecodeErrorMode decode_error_mode) {
- RTC_DCHECK(construction_thread_.CalledOnValidThread());
- // TODO(tommi): This method must only be called when the decoder thread
- // is not running and we don't need to hold this lock.
- rtc::CritScope cs(&receive_crit_);
+ RTC_DCHECK_RUN_ON(&construction_thread_);
+ RTC_DCHECK(!IsDecoderThreadRunning());
switch (robustnessMode) {
case VideoCodingModule::kNone:
_receiver.SetNackMode(kNoNack, -1, -1);
@@ -434,24 +493,40 @@
}
void VideoReceiver::SetDecodeErrorMode(VCMDecodeErrorMode decode_error_mode) {
- rtc::CritScope cs(&receive_crit_);
+ RTC_DCHECK_RUN_ON(&construction_thread_);
+ RTC_DCHECK(!IsDecoderThreadRunning());
_receiver.SetDecodeErrorMode(decode_error_mode);
}
void VideoReceiver::SetNackSettings(size_t max_nack_list_size,
int max_packet_age_to_nack,
int max_incomplete_time_ms) {
- if (max_nack_list_size != 0) {
- rtc::CritScope cs(&process_crit_);
+ RTC_DCHECK_RUN_ON(&construction_thread_);
+ RTC_DCHECK(!IsDecoderThreadRunning());
+
+ if (max_nack_list_size != 0)
max_nack_list_size_ = max_nack_list_size;
- }
_receiver.SetNackSettings(max_nack_list_size, max_packet_age_to_nack,
max_incomplete_time_ms);
}
int VideoReceiver::SetMinReceiverDelay(int desired_delay_ms) {
+ RTC_DCHECK_RUN_ON(&construction_thread_);
+ RTC_DCHECK(!IsDecoderThreadRunning());
+ // TODO(tommi): Is the method only used by tests? Maybe could be offered
+ // via a test only subclass?
+ // Info from Stefan: If it is indeed only used by tests I think it's just that
+ // it hasn't been cleaned up when the calling code was cleaned up.
return _receiver.SetMinReceiverDelay(desired_delay_ms);
}
+bool VideoReceiver::IsDecoderThreadRunning() {
+#if RTC_DCHECK_IS_ON
+ return decoder_thread_is_running_;
+#else
+ return true;
+#endif
+}
+
} // namespace vcm
} // namespace webrtc