Wire up VCMTiming in FrameBuffer2.
The VCMTiming class is now used to set the render time for frames.
BUG=webrtc:5514
R=stefan@webrtc.org
Review URL: https://codereview.webrtc.org/2138873003 .
Cr-Commit-Position: refs/heads/master@{#13621}
diff --git a/webrtc/modules/video_coding/frame_buffer2.cc b/webrtc/modules/video_coding/frame_buffer2.cc
index 53b30c9..dab7832 100644
--- a/webrtc/modules/video_coding/frame_buffer2.cc
+++ b/webrtc/modules/video_coding/frame_buffer2.cc
@@ -42,13 +42,15 @@
FrameBuffer::FrameBuffer(Clock* clock,
VCMJitterEstimator* jitter_estimator,
- const VCMTiming* timing)
+ VCMTiming* timing)
: clock_(clock),
frame_inserted_event_(false, false),
jitter_estimator_(jitter_estimator),
timing_(timing),
+ inter_frame_delay_(clock_->TimeInMilliseconds()),
newest_picture_id_(-1),
- stopped_(false) {}
+ stopped_(false),
+ protection_mode_(kProtectionNack) {}
std::unique_ptr<FrameObject> FrameBuffer::NextFrame(int64_t max_wait_time_ms) {
int64_t latest_return_time = clock_->TimeInMilliseconds() + max_wait_time_ms;
@@ -56,7 +58,7 @@
int64_t wait_ms = max_wait_time_ms;
while (true) {
std::map<FrameKey, std::unique_ptr<FrameObject>, FrameComp>::iterator
- next_frame;
+ next_frame_it;
{
rtc::CritScope lock(&crit_);
frame_inserted_event_.Reset();
@@ -65,14 +67,18 @@
now = clock_->TimeInMilliseconds();
wait_ms = max_wait_time_ms;
- next_frame = frames_.end();
+ next_frame_it = frames_.end();
for (auto frame_it = frames_.begin(); frame_it != frames_.end();
++frame_it) {
const FrameObject& frame = *frame_it->second;
if (IsContinuous(frame)) {
- next_frame = frame_it;
- int64_t render_time = timing_->RenderTimeMs(frame.timestamp, now);
+ next_frame_it = frame_it;
+ int64_t render_time =
+ next_frame_it->second->RenderTime() == -1
+ ? timing_->RenderTimeMs(frame.timestamp, now)
+ : next_frame_it->second->RenderTime();
wait_ms = timing_->MaxWaitingTime(render_time, now);
+ frame_it->second->SetRenderTime(render_time);
// This will cause the frame buffer to prefer high framerate rather
// than high resolution in the case of the decoder not decoding fast
@@ -85,19 +91,30 @@
}
}
- // If the timout occures, return. Otherwise a new frame has been inserted
- // and the best frame to decode next will be selected again.
wait_ms = std::min<int64_t>(wait_ms, latest_return_time - now);
wait_ms = std::max<int64_t>(wait_ms, 0);
+ // If the timeout occurs, return. Otherwise a new frame has been inserted
+ // and the best frame to decode next will be selected again.
if (!frame_inserted_event_.Wait(wait_ms)) {
rtc::CritScope lock(&crit_);
- if (next_frame != frames_.end()) {
- // TODO(philipel): update jitter estimator with correct values.
- jitter_estimator_->UpdateEstimate(100, 100);
+ if (next_frame_it != frames_.end()) {
+ int64_t received_timestamp = next_frame_it->second->ReceivedTime();
+ uint32_t timestamp = next_frame_it->second->Timestamp();
- decoded_frames_.insert(next_frame->first);
- std::unique_ptr<FrameObject> frame = std::move(next_frame->second);
- frames_.erase(frames_.begin(), ++next_frame);
+ int64_t frame_delay;
+ if (inter_frame_delay_.CalculateDelay(timestamp, &frame_delay,
+ received_timestamp)) {
+ jitter_estimator_->UpdateEstimate(frame_delay,
+ next_frame_it->second->size);
+ }
+ float rtt_mult = protection_mode_ == kProtectionNackFEC ? 0.0 : 1.0;
+ timing_->SetJitterDelay(jitter_estimator_->GetJitterEstimate(rtt_mult));
+ timing_->UpdateCurrentDelay(next_frame_it->second->RenderTime(),
+ clock_->TimeInMilliseconds());
+
+ decoded_frames_.insert(next_frame_it->first);
+ std::unique_ptr<FrameObject> frame = std::move(next_frame_it->second);
+ frames_.erase(frames_.begin(), ++next_frame_it);
return frame;
} else {
return std::unique_ptr<FrameObject>();
@@ -106,6 +123,11 @@
}
}
+void FrameBuffer::SetProtectionMode(VCMVideoProtection mode) {
+ rtc::CritScope lock(&crit_);
+ protection_mode_ = mode;
+}
+
void FrameBuffer::Start() {
rtc::CritScope lock(&crit_);
stopped_ = false;
@@ -119,6 +141,7 @@
void FrameBuffer::InsertFrame(std::unique_ptr<FrameObject> frame) {
rtc::CritScope lock(&crit_);
+ // If |newest_picture_id_| is -1 then this is the first frame we received.
if (newest_picture_id_ == -1)
newest_picture_id_ = frame->picture_id;
@@ -129,7 +152,7 @@
while (decoded_frames_.size() > kMaxNumHistoryFrames)
decoded_frames_.erase(decoded_frames_.begin());
- // Remove frames that are too old, |kMaxNumHistoryFrames|.
+ // Remove frames that are too old.
uint16_t old_picture_id = Subtract<1 << 16>(newest_picture_id_, kMaxFrameAge);
auto old_decoded_it =
decoded_frames_.lower_bound(FrameKey(old_picture_id, 0));
diff --git a/webrtc/modules/video_coding/frame_buffer2.h b/webrtc/modules/video_coding/frame_buffer2.h
index 3201f62..0af2bf9 100644
--- a/webrtc/modules/video_coding/frame_buffer2.h
+++ b/webrtc/modules/video_coding/frame_buffer2.h
@@ -21,6 +21,8 @@
#include "webrtc/base/criticalsection.h"
#include "webrtc/base/event.h"
#include "webrtc/base/thread_annotations.h"
+#include "webrtc/modules/video_coding/include/video_coding_defines.h"
+#include "webrtc/modules/video_coding/inter_frame_delay.h"
namespace webrtc {
@@ -36,7 +38,7 @@
public:
FrameBuffer(Clock* clock,
VCMJitterEstimator* jitter_estimator,
- const VCMTiming* timing);
+ VCMTiming* timing);
// Insert a frame into the frame buffer.
void InsertFrame(std::unique_ptr<FrameObject> frame);
@@ -46,6 +48,12 @@
// unique ptr if there is no available frame for decoding.
std::unique_ptr<FrameObject> NextFrame(int64_t max_wait_time_ms);
+ // Tells the FrameBuffer which protection mode that is in use. Affects
+ // the frame timing.
+ // TODO(philipel): Remove this when new timing calculations has been
+ // implemented.
+ void SetProtectionMode(VCMVideoProtection mode);
+
// Start the frame buffer, has no effect if the frame buffer is started.
// The frame buffer is started upon construction.
void Start();
@@ -78,10 +86,12 @@
rtc::CriticalSection crit_;
Clock* const clock_;
rtc::Event frame_inserted_event_;
- VCMJitterEstimator* const jitter_estimator_;
- const VCMTiming* const timing_;
+ VCMJitterEstimator* const jitter_estimator_ GUARDED_BY(crit_);
+ VCMTiming* const timing_ GUARDED_BY(crit_);
+ VCMInterFrameDelay inter_frame_delay_ GUARDED_BY(crit_);
int newest_picture_id_ GUARDED_BY(crit_);
bool stopped_ GUARDED_BY(crit_);
+ VCMVideoProtection protection_mode_ GUARDED_BY(crit_);
RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(FrameBuffer);
};
diff --git a/webrtc/modules/video_coding/frame_buffer2_unittest.cc b/webrtc/modules/video_coding/frame_buffer2_unittest.cc
index 5c81d02..674ae29 100644
--- a/webrtc/modules/video_coding/frame_buffer2_unittest.cc
+++ b/webrtc/modules/video_coding/frame_buffer2_unittest.cc
@@ -70,6 +70,7 @@
void(int64_t frameDelayMs,
uint32_t frameSizeBytes,
bool incompleteFrame));
+ MOCK_METHOD1(GetJitterEstimate, int(double rttMultiplier));
};
class FrameObjectFake : public FrameObject {
@@ -178,7 +179,7 @@
SimulatedClock clock_;
VCMTimingFake timing_;
- VCMJitterEstimatorMock jitter_estimator_;
+ ::testing::NiceMock<VCMJitterEstimatorMock> jitter_estimator_;
FrameBuffer buffer_;
std::vector<std::unique_ptr<FrameObject>> frames_;
Random rand_;
@@ -336,5 +337,19 @@
CheckNoFrame(2);
}
+TEST_F(TestFrameBuffer2, ProtectionMode) {
+ uint16_t pid = Rand();
+ uint32_t ts = Rand();
+
+ EXPECT_CALL(jitter_estimator_, GetJitterEstimate(1.0));
+ InsertFrame(pid, 0, ts, false);
+ ExtractFrame();
+
+ buffer_.SetProtectionMode(kProtectionNackFEC);
+ EXPECT_CALL(jitter_estimator_, GetJitterEstimate(0.0));
+ InsertFrame(pid + 1, 0, ts, false);
+ ExtractFrame();
+}
+
} // namespace video_coding
} // namespace webrtc
diff --git a/webrtc/modules/video_coding/jitter_estimator.h b/webrtc/modules/video_coding/jitter_estimator.h
index a7b4b3e..7ef050f 100644
--- a/webrtc/modules/video_coding/jitter_estimator.h
+++ b/webrtc/modules/video_coding/jitter_estimator.h
@@ -50,7 +50,7 @@
// - rttMultiplier : RTT param multiplier (when applicable).
//
// Return value : Jitter estimate in milliseconds
- int GetJitterEstimate(double rttMultiplier);
+ virtual int GetJitterEstimate(double rttMultiplier);
// Updates the nack counter.
void FrameNacked();