blob: 9913f3dcbd18aaa7b30b0ea33d6d6b69f3517284 [file] [log] [blame]
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +00001/*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10#include <stdio.h>
11
12#include <deque>
13#include <map>
14
pbos@webrtc.org69215d82013-07-10 15:02:02 +000015#include "testing/gtest/include/gtest/gtest.h"
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000016
kwiberg@webrtc.org00b8f6b2015-02-26 14:34:55 +000017#include "webrtc/base/scoped_ptr.h"
pbos@webrtc.org38344ed2014-09-24 06:05:00 +000018#include "webrtc/base/thread_annotations.h"
pbos@webrtc.org16e03b72013-10-28 16:32:01 +000019#include "webrtc/call.h"
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000020#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
21#include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h"
22#include "webrtc/system_wrappers/interface/clock.h"
sprang@webrtc.org131bea82015-02-18 12:46:06 +000023#include "webrtc/system_wrappers/interface/cpu_info.h"
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000024#include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
25#include "webrtc/system_wrappers/interface/event_wrapper.h"
pbos@webrtc.org94015242013-10-16 11:05:37 +000026#include "webrtc/system_wrappers/interface/sleep.h"
pbos@webrtc.org994d0b72014-06-27 08:47:52 +000027#include "webrtc/test/call_test.h"
pbos@webrtc.org16e03b72013-10-28 16:32:01 +000028#include "webrtc/test/direct_transport.h"
pbos@webrtc.orgf577ae92014-03-19 08:43:57 +000029#include "webrtc/test/encoder_settings.h"
30#include "webrtc/test/fake_encoder.h"
sprang@webrtc.org131bea82015-02-18 12:46:06 +000031#include "webrtc/test/frame_generator.h"
pbos@webrtc.org16e03b72013-10-28 16:32:01 +000032#include "webrtc/test/frame_generator_capturer.h"
pbos@webrtc.org16e03b72013-10-28 16:32:01 +000033#include "webrtc/test/statistics.h"
pbos@webrtc.orgf577ae92014-03-19 08:43:57 +000034#include "webrtc/test/testsupport/fileutils.h"
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000035#include "webrtc/typedefs.h"
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000036
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000037namespace webrtc {
38
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +000039static const int kFullStackTestDurationSecs = 60;
pbos@webrtc.orgb613b5a2013-12-03 10:13:04 +000040
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000041struct FullStackTestParams {
42 const char* test_label;
43 struct {
44 const char* name;
pbos@webrtc.orgf3f13582013-07-09 14:04:46 +000045 size_t width, height;
46 int fps;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000047 } clip;
sprang@webrtc.org131bea82015-02-18 12:46:06 +000048 bool screenshare;
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +000049 int min_bitrate_bps;
50 int target_bitrate_bps;
51 int max_bitrate_bps;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000052 double avg_psnr_threshold;
53 double avg_ssim_threshold;
sprang@webrtc.org131bea82015-02-18 12:46:06 +000054 int test_durations_secs;
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +000055 FakeNetworkPipe::Config link;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000056};
57
pbos@webrtc.org994d0b72014-06-27 08:47:52 +000058class FullStackTest : public test::CallTest {
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000059 protected:
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +000060 void RunTest(const FullStackTestParams& params);
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000061};
62
pbos@webrtc.org74fa4892013-08-23 09:19:30 +000063class VideoAnalyzer : public PacketReceiver,
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000064 public newapi::Transport,
pbos@webrtc.org74fa4892013-08-23 09:19:30 +000065 public VideoRenderer,
66 public VideoSendStreamInput {
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000067 public:
pbos@webrtc.org74fa4892013-08-23 09:19:30 +000068 VideoAnalyzer(VideoSendStreamInput* input,
69 Transport* transport,
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000070 const char* test_label,
71 double avg_psnr_threshold,
72 double avg_ssim_threshold,
pbos@webrtc.org94015242013-10-16 11:05:37 +000073 int duration_frames)
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000074 : input_(input),
75 transport_(transport),
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000076 receiver_(NULL),
77 test_label_(test_label),
sprang@webrtc.org131bea82015-02-18 12:46:06 +000078 frames_to_process_(duration_frames),
79 frames_recorded_(0),
80 frames_processed_(0),
pbos@webrtc.org94015242013-10-16 11:05:37 +000081 dropped_frames_(0),
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000082 last_render_time_(0),
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +000083 rtp_timestamp_delta_(0),
84 crit_(CriticalSectionWrapper::CreateCriticalSection()),
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000085 avg_psnr_threshold_(avg_psnr_threshold),
86 avg_ssim_threshold_(avg_ssim_threshold),
pbos@webrtc.org94015242013-10-16 11:05:37 +000087 comparison_lock_(CriticalSectionWrapper::CreateCriticalSection()),
sprang@webrtc.org131bea82015-02-18 12:46:06 +000088 comparison_available_event_(EventWrapper::Create()),
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +000089 done_(EventWrapper::Create()) {
sprang@webrtc.org131bea82015-02-18 12:46:06 +000090 // Create thread pool for CPU-expensive PSNR/SSIM calculations.
91
92 // Try to use about as many threads as cores, but leave kMinCoresLeft alone,
93 // so that we don't accidentally starve "real" worker threads (codec etc).
94 // Also, don't allocate more than kMaxComparisonThreads, even if there are
95 // spare cores.
96
97 uint32_t num_cores = CpuInfo::DetectNumberOfCores();
98 assert(num_cores >= 1);
99 static const uint32_t kMinCoresLeft = 4;
sprang@webrtc.org343096a2015-02-23 08:34:17 +0000100 static const uint32_t kMaxComparisonThreads = 8;
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000101
102 if (num_cores <= kMinCoresLeft) {
103 num_cores = 1;
104 } else {
105 num_cores -= kMinCoresLeft;
106 num_cores = std::min(num_cores, kMaxComparisonThreads);
107 }
108
109 for (uint32_t i = 0; i < num_cores; ++i) {
tommi@webrtc.org361981f2015-03-19 14:44:18 +0000110 rtc::scoped_ptr<ThreadWrapper> thread =
tommi@webrtc.org90a1cb42015-03-22 14:33:54 +0000111 ThreadWrapper::CreateThread(&FrameComparisonThread, this);
pbos@webrtc.org86639732015-03-13 00:06:21 +0000112 EXPECT_TRUE(thread->Start());
tommi@webrtc.org361981f2015-03-19 14:44:18 +0000113 comparison_thread_pool_.push_back(thread.release());
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000114 }
pbos@webrtc.org94015242013-10-16 11:05:37 +0000115 }
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000116
117 ~VideoAnalyzer() {
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000118 for (ThreadWrapper* thread : comparison_thread_pool_) {
119 EXPECT_TRUE(thread->Stop());
120 delete thread;
121 }
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000122 }
123
pbos@webrtc.org94015242013-10-16 11:05:37 +0000124 virtual void SetReceiver(PacketReceiver* receiver) { receiver_ = receiver; }
125
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000126 DeliveryStatus DeliverPacket(const uint8_t* packet, size_t length) override {
kwiberg@webrtc.org00b8f6b2015-02-26 14:34:55 +0000127 rtc::scoped_ptr<RtpHeaderParser> parser(RtpHeaderParser::Create());
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000128 RTPHeader header;
pbos@webrtc.org62bafae2014-07-08 12:10:51 +0000129 parser->Parse(packet, length, &header);
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000130 {
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000131 CriticalSectionScoped lock(crit_.get());
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000132 recv_times_[header.timestamp - rtp_timestamp_delta_] =
133 Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
134 }
135
136 return receiver_->DeliverPacket(packet, length);
137 }
138
perkj@webrtc.orgaf612d52015-03-18 09:51:05 +0000139 void IncomingCapturedFrame(const I420VideoFrame& video_frame) override {
pbos@webrtc.org443ad402015-03-20 07:34:28 +0000140 I420VideoFrame copy = video_frame;
141 copy.set_timestamp(copy.ntp_time_ms() * 90);
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000142
143 {
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000144 CriticalSectionScoped lock(crit_.get());
pbos@webrtc.org443ad402015-03-20 07:34:28 +0000145 if (first_send_frame_.IsZeroSize() && rtp_timestamp_delta_ == 0)
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000146 first_send_frame_ = copy;
147
148 frames_.push_back(copy);
149 }
150
perkj@webrtc.orgaf612d52015-03-18 09:51:05 +0000151 input_->IncomingCapturedFrame(video_frame);
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000152 }
153
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000154 bool SendRtp(const uint8_t* packet, size_t length) override {
kwiberg@webrtc.org00b8f6b2015-02-26 14:34:55 +0000155 rtc::scoped_ptr<RtpHeaderParser> parser(RtpHeaderParser::Create());
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000156 RTPHeader header;
pbos@webrtc.org62bafae2014-07-08 12:10:51 +0000157 parser->Parse(packet, length, &header);
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000158
159 {
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000160 CriticalSectionScoped lock(crit_.get());
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000161 if (rtp_timestamp_delta_ == 0) {
162 rtp_timestamp_delta_ =
pbos@webrtc.org443ad402015-03-20 07:34:28 +0000163 header.timestamp - first_send_frame_.timestamp();
164 first_send_frame_.Reset();
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000165 }
pbos@webrtc.org7fb9ce02013-08-05 09:29:50 +0000166 send_times_[header.timestamp - rtp_timestamp_delta_] =
167 Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000168 }
169
pbos@webrtc.org27326b62013-11-20 12:17:04 +0000170 return transport_->SendRtp(packet, length);
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000171 }
172
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000173 bool SendRtcp(const uint8_t* packet, size_t length) override {
pbos@webrtc.org27326b62013-11-20 12:17:04 +0000174 return transport_->SendRtcp(packet, length);
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000175 }
176
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000177 void RenderFrame(const I420VideoFrame& video_frame,
178 int time_to_render_ms) override {
pbos@webrtc.org94015242013-10-16 11:05:37 +0000179 int64_t render_time_ms =
180 Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000181 uint32_t send_timestamp = video_frame.timestamp() - rtp_timestamp_delta_;
182
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000183 CriticalSectionScoped lock(crit_.get());
pbos@webrtc.org443ad402015-03-20 07:34:28 +0000184
185 while (frames_.front().timestamp() < send_timestamp) {
186 AddFrameComparison(frames_.front(), last_rendered_frame_, true,
187 render_time_ms);
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000188 frames_.pop_front();
pbos@webrtc.org94015242013-10-16 11:05:37 +0000189 }
190
pbos@webrtc.org443ad402015-03-20 07:34:28 +0000191 I420VideoFrame reference_frame = frames_.front();
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000192 frames_.pop_front();
pbos@webrtc.org443ad402015-03-20 07:34:28 +0000193 assert(!reference_frame.IsZeroSize());
194 EXPECT_EQ(reference_frame.timestamp(), send_timestamp);
195 assert(reference_frame.timestamp() == send_timestamp);
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000196
pbos@webrtc.org443ad402015-03-20 07:34:28 +0000197 AddFrameComparison(reference_frame, video_frame, false, render_time_ms);
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000198
pbos@webrtc.org443ad402015-03-20 07:34:28 +0000199 last_rendered_frame_ = video_frame;
pbos@webrtc.org94015242013-10-16 11:05:37 +0000200 }
201
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000202 bool IsTextureSupported() const override { return false; }
pbos@webrtc.org0d852d52015-02-09 15:14:36 +0000203
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000204 void Wait() {
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000205 // Frame comparisons can be very expensive. Wait for test to be done, but
206 // at time-out check if frames_processed is going up. If so, give it more
207 // time, otherwise fail. Hopefully this will reduce test flakiness.
208
209 int last_frames_processed = -1;
210 EventTypeWrapper eventType;
211 while ((eventType = done_->Wait(FullStackTest::kDefaultTimeoutMs)) !=
212 kEventSignaled) {
213 int frames_processed;
214 {
215 CriticalSectionScoped crit(comparison_lock_.get());
216 frames_processed = frames_processed_;
217 }
218 if (last_frames_processed == -1) {
219 last_frames_processed = frames_processed;
220 continue;
221 }
222 ASSERT_GT(frames_processed, last_frames_processed)
223 << "Analyzer stalled while waiting for test to finish.";
224 last_frames_processed = frames_processed;
225 }
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000226 }
pbos@webrtc.org94015242013-10-16 11:05:37 +0000227
228 VideoSendStreamInput* input_;
229 Transport* transport_;
230 PacketReceiver* receiver_;
231
232 private:
233 struct FrameComparison {
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000234 FrameComparison()
235 : dropped(false), send_time_ms(0), recv_time_ms(0), render_time_ms(0) {}
236
pbos@webrtc.org443ad402015-03-20 07:34:28 +0000237 FrameComparison(const I420VideoFrame& reference,
238 const I420VideoFrame& render,
pbos@webrtc.org94015242013-10-16 11:05:37 +0000239 bool dropped,
240 int64_t send_time_ms,
241 int64_t recv_time_ms,
242 int64_t render_time_ms)
pbos@webrtc.org443ad402015-03-20 07:34:28 +0000243 : reference(reference),
244 render(render),
245 dropped(dropped),
pbos@webrtc.org94015242013-10-16 11:05:37 +0000246 send_time_ms(send_time_ms),
247 recv_time_ms(recv_time_ms),
pbos@webrtc.org443ad402015-03-20 07:34:28 +0000248 render_time_ms(render_time_ms) {}
pbos@webrtc.org94015242013-10-16 11:05:37 +0000249
250 I420VideoFrame reference;
251 I420VideoFrame render;
252 bool dropped;
253 int64_t send_time_ms;
254 int64_t recv_time_ms;
255 int64_t render_time_ms;
256 };
257
pbos@webrtc.org443ad402015-03-20 07:34:28 +0000258 void AddFrameComparison(const I420VideoFrame& reference,
259 const I420VideoFrame& render,
pbos@webrtc.org94015242013-10-16 11:05:37 +0000260 bool dropped,
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000261 int64_t render_time_ms)
262 EXCLUSIVE_LOCKS_REQUIRED(crit_) {
pbos@webrtc.org443ad402015-03-20 07:34:28 +0000263 int64_t send_time_ms = send_times_[reference.timestamp()];
264 send_times_.erase(reference.timestamp());
265 int64_t recv_time_ms = recv_times_[reference.timestamp()];
266 recv_times_.erase(reference.timestamp());
pbos@webrtc.org94015242013-10-16 11:05:37 +0000267
268 CriticalSectionScoped crit(comparison_lock_.get());
269 comparisons_.push_back(FrameComparison(reference,
270 render,
271 dropped,
272 send_time_ms,
273 recv_time_ms,
274 render_time_ms));
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000275 comparison_available_event_->Set();
pbos@webrtc.org94015242013-10-16 11:05:37 +0000276 }
277
278 static bool FrameComparisonThread(void* obj) {
279 return static_cast<VideoAnalyzer*>(obj)->CompareFrames();
280 }
281
282 bool CompareFrames() {
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000283 if (AllFramesRecorded())
284 return false;
pbos@webrtc.org94015242013-10-16 11:05:37 +0000285
286 I420VideoFrame reference;
287 I420VideoFrame render;
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000288 FrameComparison comparison;
pbos@webrtc.org94015242013-10-16 11:05:37 +0000289
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000290 if (!PopComparison(&comparison)) {
291 // Wait until new comparison task is available, or test is done.
292 // If done, wake up remaining threads waiting.
293 comparison_available_event_->Wait(1000);
294 if (AllFramesRecorded()) {
295 comparison_available_event_->Set();
pbos@webrtc.org94015242013-10-16 11:05:37 +0000296 return false;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000297 }
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000298 return true; // Try again.
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000299 }
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000300
301 PerformFrameComparison(comparison);
302
303 if (FrameProcessed()) {
304 PrintResults();
305 done_->Set();
306 comparison_available_event_->Set();
307 return false;
308 }
309
310 return true;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000311 }
312
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000313 bool PopComparison(FrameComparison* comparison) {
314 CriticalSectionScoped crit(comparison_lock_.get());
315 // If AllFramesRecorded() is true, it means we have already popped
316 // frames_to_process_ frames from comparisons_, so there is no more work
317 // for this thread to be done. frames_processed_ might still be lower if
318 // all comparisons are not done, but those frames are currently being
319 // worked on by other threads.
320 if (comparisons_.empty() || AllFramesRecorded())
321 return false;
322
pbos@webrtc.org443ad402015-03-20 07:34:28 +0000323 *comparison = comparisons_.front();
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000324 comparisons_.pop_front();
325
326 FrameRecorded();
327 return true;
328 }
329
330 // Increment counter for number of frames received for comparison.
331 void FrameRecorded() {
332 CriticalSectionScoped crit(comparison_lock_.get());
333 ++frames_recorded_;
334 }
335
336 // Returns true if all frames to be compared have been taken from the queue.
337 bool AllFramesRecorded() {
338 CriticalSectionScoped crit(comparison_lock_.get());
339 assert(frames_recorded_ <= frames_to_process_);
340 return frames_recorded_ == frames_to_process_;
341 }
342
343 // Increase count of number of frames processed. Returns true if this was the
344 // last frame to be processed.
345 bool FrameProcessed() {
346 CriticalSectionScoped crit(comparison_lock_.get());
347 ++frames_processed_;
348 assert(frames_processed_ <= frames_to_process_);
349 return frames_processed_ == frames_to_process_;
350 }
351
352 void PrintResults() {
353 CriticalSectionScoped crit(comparison_lock_.get());
354 PrintResult("psnr", psnr_, " dB");
355 PrintResult("ssim", ssim_, "");
356 PrintResult("sender_time", sender_time_, " ms");
357 printf("RESULT dropped_frames: %s = %d frames\n", test_label_,
358 dropped_frames_);
359 PrintResult("receiver_time", receiver_time_, " ms");
360 PrintResult("total_delay_incl_network", end_to_end_, " ms");
361 PrintResult("time_between_rendered_frames", rendered_delta_, " ms");
362 EXPECT_GT(psnr_.Mean(), avg_psnr_threshold_);
363 EXPECT_GT(ssim_.Mean(), avg_ssim_threshold_);
364 }
365
366 void PerformFrameComparison(const FrameComparison& comparison) {
367 // Perform expensive psnr and ssim calculations while not holding lock.
magjed@webrtc.org2056ee32015-03-16 13:46:52 +0000368 double psnr = I420PSNR(&comparison.reference, &comparison.render);
369 double ssim = I420SSIM(&comparison.reference, &comparison.render);
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000370
371 CriticalSectionScoped crit(comparison_lock_.get());
372 psnr_.AddSample(psnr);
373 ssim_.AddSample(ssim);
374 if (comparison.dropped) {
pbos@webrtc.org94015242013-10-16 11:05:37 +0000375 ++dropped_frames_;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000376 return;
pbos@webrtc.org94015242013-10-16 11:05:37 +0000377 }
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000378 if (last_render_time_ != 0)
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000379 rendered_delta_.AddSample(comparison.render_time_ms - last_render_time_);
380 last_render_time_ = comparison.render_time_ms;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000381
perkj@webrtc.orgaf612d52015-03-18 09:51:05 +0000382 int64_t input_time_ms = comparison.reference.ntp_time_ms();
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000383 sender_time_.AddSample(comparison.send_time_ms - input_time_ms);
384 receiver_time_.AddSample(comparison.render_time_ms -
385 comparison.recv_time_ms);
386 end_to_end_.AddSample(comparison.render_time_ms - input_time_ms);
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000387 }
388
389 void PrintResult(const char* result_type,
390 test::Statistics stats,
391 const char* unit) {
392 printf("RESULT %s: %s = {%f, %f}%s\n",
393 result_type,
394 test_label_,
395 stats.Mean(),
396 stats.StandardDeviation(),
397 unit);
398 }
399
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000400 const char* const test_label_;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000401 test::Statistics sender_time_;
402 test::Statistics receiver_time_;
403 test::Statistics psnr_;
404 test::Statistics ssim_;
405 test::Statistics end_to_end_;
406 test::Statistics rendered_delta_;
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000407 const int frames_to_process_;
408 int frames_recorded_;
409 int frames_processed_;
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000410 int dropped_frames_;
411 int64_t last_render_time_;
412 uint32_t rtp_timestamp_delta_;
413
kwiberg@webrtc.org00b8f6b2015-02-26 14:34:55 +0000414 const rtc::scoped_ptr<CriticalSectionWrapper> crit_;
pbos@webrtc.org443ad402015-03-20 07:34:28 +0000415 std::deque<I420VideoFrame> frames_ GUARDED_BY(crit_);
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000416 I420VideoFrame last_rendered_frame_ GUARDED_BY(crit_);
417 std::map<uint32_t, int64_t> send_times_ GUARDED_BY(crit_);
418 std::map<uint32_t, int64_t> recv_times_ GUARDED_BY(crit_);
pbos@webrtc.org443ad402015-03-20 07:34:28 +0000419 I420VideoFrame first_send_frame_ GUARDED_BY(crit_);
pbos@webrtc.orgb35b1362014-10-20 09:14:38 +0000420 const double avg_psnr_threshold_;
421 const double avg_ssim_threshold_;
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000422
kwiberg@webrtc.org00b8f6b2015-02-26 14:34:55 +0000423 const rtc::scoped_ptr<CriticalSectionWrapper> comparison_lock_;
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000424 std::vector<ThreadWrapper*> comparison_thread_pool_;
kwiberg@webrtc.org00b8f6b2015-02-26 14:34:55 +0000425 const rtc::scoped_ptr<EventWrapper> comparison_available_event_;
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000426 std::deque<FrameComparison> comparisons_ GUARDED_BY(comparison_lock_);
kwiberg@webrtc.org00b8f6b2015-02-26 14:34:55 +0000427 const rtc::scoped_ptr<EventWrapper> done_;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000428};
429
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000430void FullStackTest::RunTest(const FullStackTestParams& params) {
431 test::DirectTransport send_transport(params.link);
432 test::DirectTransport recv_transport(params.link);
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000433 VideoAnalyzer analyzer(NULL, &send_transport, params.test_label,
434 params.avg_psnr_threshold, params.avg_ssim_threshold,
435 params.test_durations_secs * params.clip.fps);
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000436
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000437 CreateCalls(Call::Config(&analyzer), Call::Config(&recv_transport));
mflodman@webrtc.org6879c8a2013-07-23 11:35:00 +0000438
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000439 analyzer.SetReceiver(receiver_call_->Receiver());
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000440 send_transport.SetReceiver(&analyzer);
441 recv_transport.SetReceiver(sender_call_->Receiver());
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000442
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000443 CreateSendConfig(1);
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000444
kwiberg@webrtc.org00b8f6b2015-02-26 14:34:55 +0000445 rtc::scoped_ptr<VideoEncoder> encoder(
pbos@webrtc.orgab990ae2014-09-17 09:02:25 +0000446 VideoEncoder::Create(VideoEncoder::kVp8));
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000447 send_config_.encoder_settings.encoder = encoder.get();
448 send_config_.encoder_settings.payload_name = "VP8";
449 send_config_.encoder_settings.payload_type = 124;
stefan@webrtc.orgc216b9a2014-10-14 10:38:49 +0000450 send_config_.rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000451 send_config_.rtp.rtx.ssrcs.push_back(kSendRtxSsrcs[0]);
452 send_config_.rtp.rtx.payload_type = kSendRtxPayloadType;
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000453
pbos@webrtc.orgbbe0a852014-09-19 12:30:25 +0000454 VideoStream* stream = &encoder_config_.streams[0];
pbos@webrtc.orgf577ae92014-03-19 08:43:57 +0000455 stream->width = params.clip.width;
456 stream->height = params.clip.height;
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000457 stream->min_bitrate_bps = params.min_bitrate_bps;
458 stream->target_bitrate_bps = params.target_bitrate_bps;
459 stream->max_bitrate_bps = params.max_bitrate_bps;
pbos@webrtc.orgf577ae92014-03-19 08:43:57 +0000460 stream->max_framerate = params.clip.fps;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000461
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000462 if (params.screenshare) {
463 encoder_config_.content_type = VideoEncoderConfig::kScreenshare;
464 encoder_config_.min_transmit_bitrate_bps = 400 * 1000;
465 VideoCodecVP8 vp8_settings = VideoEncoder::GetDefaultVp8Settings();
466 vp8_settings.denoisingOn = false;
467 vp8_settings.frameDroppingOn = false;
468 vp8_settings.numberOfTemporalLayers = 2;
469 encoder_config_.encoder_specific_settings = &vp8_settings;
470
471 stream->temporal_layer_thresholds_bps.clear();
472 stream->temporal_layer_thresholds_bps.push_back(stream->target_bitrate_bps);
473 }
474
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000475 CreateMatchingReceiveConfigs();
pbos@webrtc.orgbe9d2a42014-06-30 13:19:09 +0000476 receive_configs_[0].renderer = &analyzer;
stefan@webrtc.orgc216b9a2014-10-14 10:38:49 +0000477 receive_configs_[0].rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000478 receive_configs_[0].rtp.rtx[kSendRtxPayloadType].ssrc = kSendRtxSsrcs[0];
479 receive_configs_[0].rtp.rtx[kSendRtxPayloadType].payload_type =
sprang@webrtc.org343096a2015-02-23 08:34:17 +0000480 kSendRtxPayloadType;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000481
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000482 CreateStreams();
483 analyzer.input_ = send_stream_->Input();
484
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000485 if (params.screenshare) {
486 std::vector<std::string> slides;
487 slides.push_back(test::ResourcePath("web_screenshot_1850_1110", "yuv"));
488 slides.push_back(test::ResourcePath("presentation_1850_1110", "yuv"));
489 slides.push_back(test::ResourcePath("photo_1850_1110", "yuv"));
490 slides.push_back(test::ResourcePath("difficult_photo_1850_1110", "yuv"));
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000491
kwiberg@webrtc.org00b8f6b2015-02-26 14:34:55 +0000492 rtc::scoped_ptr<test::FrameGenerator> frame_generator(
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000493 test::FrameGenerator::CreateFromYuvFile(
494 slides, 1850, 1110,
495 10 * params.clip.fps) // Cycle image every 10 seconds.
496 );
497 frame_generator_capturer_.reset(new test::FrameGeneratorCapturer(
498 Clock::GetRealTimeClock(), &analyzer, frame_generator.release(),
499 params.clip.fps));
500 ASSERT_TRUE(frame_generator_capturer_->Init());
501 } else {
502 frame_generator_capturer_.reset(
503 test::FrameGeneratorCapturer::CreateFromYuvFile(
504 &analyzer, test::ResourcePath(params.clip.name, "yuv"),
505 params.clip.width, params.clip.height, params.clip.fps,
506 Clock::GetRealTimeClock()));
507
508 ASSERT_TRUE(frame_generator_capturer_.get() != NULL)
509 << "Could not create capturer for " << params.clip.name
510 << ".yuv. Is this resource file present?";
511 }
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000512
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000513 Start();
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000514
515 analyzer.Wait();
516
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000517 send_transport.StopSending();
518 recv_transport.StopSending();
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000519
520 Stop();
521
522 DestroyStreams();
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000523}
524
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000525TEST_F(FullStackTest, ParisQcifWithoutPacketLoss) {
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000526 FullStackTestParams paris_qcif = {"net_delay_0_0_plr_0",
527 {"paris_qcif", 176, 144, 30},
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000528 false,
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000529 300000,
530 300000,
531 300000,
532 36.0,
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000533 0.96,
534 kFullStackTestDurationSecs};
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000535 RunTest(paris_qcif);
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000536}
537
538TEST_F(FullStackTest, ForemanCifWithoutPacketLoss) {
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000539 // TODO(pbos): Decide on psnr/ssim thresholds for foreman_cif.
540 FullStackTestParams foreman_cif = {"foreman_cif_net_delay_0_0_plr_0",
541 {"foreman_cif", 352, 288, 30},
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000542 false,
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000543 700000,
544 700000,
545 700000,
546 0.0,
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000547 0.0,
548 kFullStackTestDurationSecs};
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000549 RunTest(foreman_cif);
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000550}
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000551
sprang@webrtc.org343096a2015-02-23 08:34:17 +0000552TEST_F(FullStackTest, ForemanCifPlr5) {
stefan@webrtc.orgc216b9a2014-10-14 10:38:49 +0000553 FullStackTestParams foreman_cif = {"foreman_cif_delay_50_0_plr_5",
554 {"foreman_cif", 352, 288, 30},
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000555 false,
stefan@webrtc.orgc216b9a2014-10-14 10:38:49 +0000556 30000,
557 500000,
558 2000000,
559 0.0,
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000560 0.0,
561 kFullStackTestDurationSecs};
stefan@webrtc.orgc216b9a2014-10-14 10:38:49 +0000562 foreman_cif.link.loss_percent = 5;
563 foreman_cif.link.queue_delay_ms = 50;
564 RunTest(foreman_cif);
565}
566
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000567TEST_F(FullStackTest, ForemanCif500kbps) {
568 FullStackTestParams foreman_cif = {"foreman_cif_500kbps",
569 {"foreman_cif", 352, 288, 30},
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000570 false,
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000571 30000,
572 500000,
573 2000000,
574 0.0,
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000575 0.0,
576 kFullStackTestDurationSecs};
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000577 foreman_cif.link.queue_length_packets = 0;
578 foreman_cif.link.queue_delay_ms = 0;
579 foreman_cif.link.link_capacity_kbps = 500;
580 RunTest(foreman_cif);
581}
582
583TEST_F(FullStackTest, ForemanCif500kbpsLimitedQueue) {
584 FullStackTestParams foreman_cif = {"foreman_cif_500kbps_32pkts_queue",
585 {"foreman_cif", 352, 288, 30},
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000586 false,
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000587 30000,
588 500000,
589 2000000,
590 0.0,
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000591 0.0,
592 kFullStackTestDurationSecs};
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000593 foreman_cif.link.queue_length_packets = 32;
594 foreman_cif.link.queue_delay_ms = 0;
595 foreman_cif.link.link_capacity_kbps = 500;
596 RunTest(foreman_cif);
597}
598
599TEST_F(FullStackTest, ForemanCif500kbps100ms) {
600 FullStackTestParams foreman_cif = {"foreman_cif_500kbps_100ms",
601 {"foreman_cif", 352, 288, 30},
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000602 false,
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000603 30000,
604 500000,
605 2000000,
606 0.0,
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000607 0.0,
608 kFullStackTestDurationSecs};
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000609 foreman_cif.link.queue_length_packets = 0;
610 foreman_cif.link.queue_delay_ms = 100;
611 foreman_cif.link.link_capacity_kbps = 500;
612 RunTest(foreman_cif);
613}
614
615TEST_F(FullStackTest, ForemanCif500kbps100msLimitedQueue) {
616 FullStackTestParams foreman_cif = {"foreman_cif_500kbps_100ms_32pkts_queue",
617 {"foreman_cif", 352, 288, 30},
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000618 false,
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000619 30000,
620 500000,
621 2000000,
622 0.0,
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000623 0.0,
624 kFullStackTestDurationSecs};
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000625 foreman_cif.link.queue_length_packets = 32;
626 foreman_cif.link.queue_delay_ms = 100;
627 foreman_cif.link.link_capacity_kbps = 500;
628 RunTest(foreman_cif);
629}
630
631TEST_F(FullStackTest, ForemanCif1000kbps100msLimitedQueue) {
632 FullStackTestParams foreman_cif = {"foreman_cif_1000kbps_100ms_32pkts_queue",
633 {"foreman_cif", 352, 288, 30},
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000634 false,
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000635 30000,
636 2000000,
637 2000000,
638 0.0,
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000639 0.0,
640 kFullStackTestDurationSecs};
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000641 foreman_cif.link.queue_length_packets = 32;
642 foreman_cif.link.queue_delay_ms = 100;
643 foreman_cif.link.link_capacity_kbps = 1000;
644 RunTest(foreman_cif);
645}
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000646
647TEST_F(FullStackTest, ScreenshareSlides) {
648 FullStackTestParams screenshare_params = {
649 "screenshare_slides",
650 {"screenshare_slides", 1850, 1110, 5},
651 true,
652 50000,
653 100000,
654 1000000,
655 0.0,
656 0.0,
657 kFullStackTestDurationSecs};
658 RunTest(screenshare_params);
659}
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000660} // namespace webrtc