blob: 033bf4fc514f092538994cb893bda37741da6c08 [file] [log] [blame]
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +00001/*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10#include <stdio.h>
11
12#include <deque>
13#include <map>
14
pbos@webrtc.org69215d82013-07-10 15:02:02 +000015#include "testing/gtest/include/gtest/gtest.h"
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000016
kwiberg@webrtc.org00b8f6b2015-02-26 14:34:55 +000017#include "webrtc/base/scoped_ptr.h"
pbos@webrtc.org38344ed2014-09-24 06:05:00 +000018#include "webrtc/base/thread_annotations.h"
pbos@webrtc.org16e03b72013-10-28 16:32:01 +000019#include "webrtc/call.h"
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000020#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
21#include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h"
22#include "webrtc/system_wrappers/interface/clock.h"
sprang@webrtc.org131bea82015-02-18 12:46:06 +000023#include "webrtc/system_wrappers/interface/cpu_info.h"
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000024#include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
25#include "webrtc/system_wrappers/interface/event_wrapper.h"
pbos@webrtc.org94015242013-10-16 11:05:37 +000026#include "webrtc/system_wrappers/interface/sleep.h"
pbos@webrtc.org994d0b72014-06-27 08:47:52 +000027#include "webrtc/test/call_test.h"
pbos@webrtc.org16e03b72013-10-28 16:32:01 +000028#include "webrtc/test/direct_transport.h"
pbos@webrtc.orgf577ae92014-03-19 08:43:57 +000029#include "webrtc/test/encoder_settings.h"
30#include "webrtc/test/fake_encoder.h"
sprang@webrtc.org131bea82015-02-18 12:46:06 +000031#include "webrtc/test/frame_generator.h"
pbos@webrtc.org16e03b72013-10-28 16:32:01 +000032#include "webrtc/test/frame_generator_capturer.h"
pbos@webrtc.org16e03b72013-10-28 16:32:01 +000033#include "webrtc/test/statistics.h"
pbos@webrtc.orgf577ae92014-03-19 08:43:57 +000034#include "webrtc/test/testsupport/fileutils.h"
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000035#include "webrtc/typedefs.h"
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000036
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000037namespace webrtc {
38
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +000039static const int kFullStackTestDurationSecs = 60;
pbos@webrtc.orgb613b5a2013-12-03 10:13:04 +000040
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000041struct FullStackTestParams {
42 const char* test_label;
43 struct {
44 const char* name;
pbos@webrtc.orgf3f13582013-07-09 14:04:46 +000045 size_t width, height;
46 int fps;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000047 } clip;
sprang@webrtc.org131bea82015-02-18 12:46:06 +000048 bool screenshare;
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +000049 int min_bitrate_bps;
50 int target_bitrate_bps;
51 int max_bitrate_bps;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000052 double avg_psnr_threshold;
53 double avg_ssim_threshold;
sprang@webrtc.org131bea82015-02-18 12:46:06 +000054 int test_durations_secs;
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +000055 FakeNetworkPipe::Config link;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000056};
57
pbos@webrtc.org994d0b72014-06-27 08:47:52 +000058class FullStackTest : public test::CallTest {
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000059 protected:
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +000060 void RunTest(const FullStackTestParams& params);
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000061};
62
pbos@webrtc.org74fa4892013-08-23 09:19:30 +000063class VideoAnalyzer : public PacketReceiver,
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000064 public newapi::Transport,
pbos@webrtc.org74fa4892013-08-23 09:19:30 +000065 public VideoRenderer,
66 public VideoSendStreamInput {
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000067 public:
pbos@webrtc.org74fa4892013-08-23 09:19:30 +000068 VideoAnalyzer(VideoSendStreamInput* input,
69 Transport* transport,
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000070 const char* test_label,
71 double avg_psnr_threshold,
72 double avg_ssim_threshold,
pbos@webrtc.org94015242013-10-16 11:05:37 +000073 int duration_frames)
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000074 : input_(input),
75 transport_(transport),
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000076 receiver_(NULL),
77 test_label_(test_label),
sprang@webrtc.org131bea82015-02-18 12:46:06 +000078 frames_to_process_(duration_frames),
79 frames_recorded_(0),
80 frames_processed_(0),
pbos@webrtc.org94015242013-10-16 11:05:37 +000081 dropped_frames_(0),
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000082 last_render_time_(0),
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +000083 rtp_timestamp_delta_(0),
84 crit_(CriticalSectionWrapper::CreateCriticalSection()),
85 first_send_frame_(NULL),
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000086 avg_psnr_threshold_(avg_psnr_threshold),
87 avg_ssim_threshold_(avg_ssim_threshold),
pbos@webrtc.org94015242013-10-16 11:05:37 +000088 comparison_lock_(CriticalSectionWrapper::CreateCriticalSection()),
sprang@webrtc.org131bea82015-02-18 12:46:06 +000089 comparison_available_event_(EventWrapper::Create()),
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +000090 done_(EventWrapper::Create()) {
sprang@webrtc.org131bea82015-02-18 12:46:06 +000091 // Create thread pool for CPU-expensive PSNR/SSIM calculations.
92
93 // Try to use about as many threads as cores, but leave kMinCoresLeft alone,
94 // so that we don't accidentally starve "real" worker threads (codec etc).
95 // Also, don't allocate more than kMaxComparisonThreads, even if there are
96 // spare cores.
97
98 uint32_t num_cores = CpuInfo::DetectNumberOfCores();
99 assert(num_cores >= 1);
100 static const uint32_t kMinCoresLeft = 4;
sprang@webrtc.org343096a2015-02-23 08:34:17 +0000101 static const uint32_t kMaxComparisonThreads = 8;
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000102
103 if (num_cores <= kMinCoresLeft) {
104 num_cores = 1;
105 } else {
106 num_cores -= kMinCoresLeft;
107 num_cores = std::min(num_cores, kMaxComparisonThreads);
108 }
109
110 for (uint32_t i = 0; i < num_cores; ++i) {
111 ThreadWrapper* thread =
112 ThreadWrapper::CreateThread(&FrameComparisonThread, this);
113 comparison_thread_pool_.push_back(thread);
114 unsigned int id;
115 EXPECT_TRUE(thread->Start(id));
116 }
pbos@webrtc.org94015242013-10-16 11:05:37 +0000117 }
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000118
119 ~VideoAnalyzer() {
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000120 for (ThreadWrapper* thread : comparison_thread_pool_) {
121 EXPECT_TRUE(thread->Stop());
122 delete thread;
123 }
pbos@webrtc.org94015242013-10-16 11:05:37 +0000124
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000125 for (I420VideoFrame* frame : frames_)
126 delete frame;
127
128 for (I420VideoFrame* frame : frame_pool_)
129 delete frame;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000130 }
131
pbos@webrtc.org94015242013-10-16 11:05:37 +0000132 virtual void SetReceiver(PacketReceiver* receiver) { receiver_ = receiver; }
133
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000134 DeliveryStatus DeliverPacket(const uint8_t* packet, size_t length) override {
kwiberg@webrtc.org00b8f6b2015-02-26 14:34:55 +0000135 rtc::scoped_ptr<RtpHeaderParser> parser(RtpHeaderParser::Create());
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000136 RTPHeader header;
pbos@webrtc.org62bafae2014-07-08 12:10:51 +0000137 parser->Parse(packet, length, &header);
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000138 {
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000139 CriticalSectionScoped lock(crit_.get());
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000140 recv_times_[header.timestamp - rtp_timestamp_delta_] =
141 Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
142 }
143
144 return receiver_->DeliverPacket(packet, length);
145 }
146
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000147 void SwapFrame(I420VideoFrame* video_frame) override {
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000148 I420VideoFrame* copy = NULL;
149 {
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000150 CriticalSectionScoped lock(crit_.get());
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000151 if (frame_pool_.size() > 0) {
152 copy = frame_pool_.front();
153 frame_pool_.pop_front();
154 }
155 }
156 if (copy == NULL)
157 copy = new I420VideoFrame();
158
pbos@webrtc.org724947b2013-12-11 16:26:16 +0000159 copy->CopyFrame(*video_frame);
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000160 copy->set_timestamp(copy->render_time_ms() * 90);
161
162 {
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000163 CriticalSectionScoped lock(crit_.get());
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000164 if (first_send_frame_ == NULL && rtp_timestamp_delta_ == 0)
165 first_send_frame_ = copy;
166
167 frames_.push_back(copy);
168 }
169
pbos@webrtc.org724947b2013-12-11 16:26:16 +0000170 input_->SwapFrame(video_frame);
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000171 }
172
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000173 bool SendRtp(const uint8_t* packet, size_t length) override {
kwiberg@webrtc.org00b8f6b2015-02-26 14:34:55 +0000174 rtc::scoped_ptr<RtpHeaderParser> parser(RtpHeaderParser::Create());
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000175 RTPHeader header;
pbos@webrtc.org62bafae2014-07-08 12:10:51 +0000176 parser->Parse(packet, length, &header);
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000177
178 {
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000179 CriticalSectionScoped lock(crit_.get());
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000180 if (rtp_timestamp_delta_ == 0) {
181 rtp_timestamp_delta_ =
182 header.timestamp - first_send_frame_->timestamp();
183 first_send_frame_ = NULL;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000184 }
pbos@webrtc.org7fb9ce02013-08-05 09:29:50 +0000185 send_times_[header.timestamp - rtp_timestamp_delta_] =
186 Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000187 }
188
pbos@webrtc.org27326b62013-11-20 12:17:04 +0000189 return transport_->SendRtp(packet, length);
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000190 }
191
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000192 bool SendRtcp(const uint8_t* packet, size_t length) override {
pbos@webrtc.org27326b62013-11-20 12:17:04 +0000193 return transport_->SendRtcp(packet, length);
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000194 }
195
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000196 void RenderFrame(const I420VideoFrame& video_frame,
197 int time_to_render_ms) override {
pbos@webrtc.org94015242013-10-16 11:05:37 +0000198 int64_t render_time_ms =
199 Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000200 uint32_t send_timestamp = video_frame.timestamp() - rtp_timestamp_delta_;
201
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000202 CriticalSectionScoped lock(crit_.get());
203 while (frames_.front()->timestamp() < send_timestamp) {
204 AddFrameComparison(
205 frames_.front(), &last_rendered_frame_, true, render_time_ms);
206 frame_pool_.push_back(frames_.front());
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000207 frames_.pop_front();
pbos@webrtc.org94015242013-10-16 11:05:37 +0000208 }
209
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000210 I420VideoFrame* reference_frame = frames_.front();
211 frames_.pop_front();
212 assert(reference_frame != NULL);
213 EXPECT_EQ(reference_frame->timestamp(), send_timestamp);
214 assert(reference_frame->timestamp() == send_timestamp);
215
216 AddFrameComparison(reference_frame, &video_frame, false, render_time_ms);
217 frame_pool_.push_back(reference_frame);
218
pbos@webrtc.org94015242013-10-16 11:05:37 +0000219 last_rendered_frame_.CopyFrame(video_frame);
220 }
221
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000222 bool IsTextureSupported() const override { return false; }
pbos@webrtc.org0d852d52015-02-09 15:14:36 +0000223
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000224 void Wait() {
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000225 // Frame comparisons can be very expensive. Wait for test to be done, but
226 // at time-out check if frames_processed is going up. If so, give it more
227 // time, otherwise fail. Hopefully this will reduce test flakiness.
228
229 int last_frames_processed = -1;
230 EventTypeWrapper eventType;
231 while ((eventType = done_->Wait(FullStackTest::kDefaultTimeoutMs)) !=
232 kEventSignaled) {
233 int frames_processed;
234 {
235 CriticalSectionScoped crit(comparison_lock_.get());
236 frames_processed = frames_processed_;
237 }
238 if (last_frames_processed == -1) {
239 last_frames_processed = frames_processed;
240 continue;
241 }
242 ASSERT_GT(frames_processed, last_frames_processed)
243 << "Analyzer stalled while waiting for test to finish.";
244 last_frames_processed = frames_processed;
245 }
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000246 }
pbos@webrtc.org94015242013-10-16 11:05:37 +0000247
248 VideoSendStreamInput* input_;
249 Transport* transport_;
250 PacketReceiver* receiver_;
251
252 private:
253 struct FrameComparison {
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000254 FrameComparison()
255 : dropped(false), send_time_ms(0), recv_time_ms(0), render_time_ms(0) {}
256
pbos@webrtc.org94015242013-10-16 11:05:37 +0000257 FrameComparison(const I420VideoFrame* reference,
258 const I420VideoFrame* render,
259 bool dropped,
260 int64_t send_time_ms,
261 int64_t recv_time_ms,
262 int64_t render_time_ms)
263 : dropped(dropped),
264 send_time_ms(send_time_ms),
265 recv_time_ms(recv_time_ms),
266 render_time_ms(render_time_ms) {
267 this->reference.CopyFrame(*reference);
268 this->render.CopyFrame(*render);
269 }
270
271 FrameComparison(const FrameComparison& compare)
272 : dropped(compare.dropped),
273 send_time_ms(compare.send_time_ms),
274 recv_time_ms(compare.recv_time_ms),
275 render_time_ms(compare.render_time_ms) {
276 this->reference.CopyFrame(compare.reference);
277 this->render.CopyFrame(compare.render);
278 }
279
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000280 void CloneStatsAndSwapFrames(FrameComparison* comparison) {
281 reference.SwapFrame(&comparison->reference);
282 render.SwapFrame(&comparison->render);
283 dropped = comparison->dropped;
284 send_time_ms = comparison->send_time_ms;
285 recv_time_ms = comparison->recv_time_ms;
286 render_time_ms = comparison->render_time_ms;
287 }
288
pbos@webrtc.org94015242013-10-16 11:05:37 +0000289 ~FrameComparison() {}
290
291 I420VideoFrame reference;
292 I420VideoFrame render;
293 bool dropped;
294 int64_t send_time_ms;
295 int64_t recv_time_ms;
296 int64_t render_time_ms;
297 };
298
299 void AddFrameComparison(const I420VideoFrame* reference,
300 const I420VideoFrame* render,
301 bool dropped,
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000302 int64_t render_time_ms)
303 EXCLUSIVE_LOCKS_REQUIRED(crit_) {
pbos@webrtc.org94015242013-10-16 11:05:37 +0000304 int64_t send_time_ms = send_times_[reference->timestamp()];
305 send_times_.erase(reference->timestamp());
306 int64_t recv_time_ms = recv_times_[reference->timestamp()];
307 recv_times_.erase(reference->timestamp());
308
309 CriticalSectionScoped crit(comparison_lock_.get());
310 comparisons_.push_back(FrameComparison(reference,
311 render,
312 dropped,
313 send_time_ms,
314 recv_time_ms,
315 render_time_ms));
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000316 comparison_available_event_->Set();
pbos@webrtc.org94015242013-10-16 11:05:37 +0000317 }
318
319 static bool FrameComparisonThread(void* obj) {
320 return static_cast<VideoAnalyzer*>(obj)->CompareFrames();
321 }
322
323 bool CompareFrames() {
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000324 if (AllFramesRecorded())
325 return false;
pbos@webrtc.org94015242013-10-16 11:05:37 +0000326
327 I420VideoFrame reference;
328 I420VideoFrame render;
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000329 FrameComparison comparison;
pbos@webrtc.org94015242013-10-16 11:05:37 +0000330
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000331 if (!PopComparison(&comparison)) {
332 // Wait until new comparison task is available, or test is done.
333 // If done, wake up remaining threads waiting.
334 comparison_available_event_->Wait(1000);
335 if (AllFramesRecorded()) {
336 comparison_available_event_->Set();
pbos@webrtc.org94015242013-10-16 11:05:37 +0000337 return false;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000338 }
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000339 return true; // Try again.
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000340 }
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000341
342 PerformFrameComparison(comparison);
343
344 if (FrameProcessed()) {
345 PrintResults();
346 done_->Set();
347 comparison_available_event_->Set();
348 return false;
349 }
350
351 return true;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000352 }
353
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000354 bool PopComparison(FrameComparison* comparison) {
355 CriticalSectionScoped crit(comparison_lock_.get());
356 // If AllFramesRecorded() is true, it means we have already popped
357 // frames_to_process_ frames from comparisons_, so there is no more work
358 // for this thread to be done. frames_processed_ might still be lower if
359 // all comparisons are not done, but those frames are currently being
360 // worked on by other threads.
361 if (comparisons_.empty() || AllFramesRecorded())
362 return false;
363
364 comparison->CloneStatsAndSwapFrames(&comparisons_.front());
365 comparisons_.pop_front();
366
367 FrameRecorded();
368 return true;
369 }
370
371 // Increment counter for number of frames received for comparison.
372 void FrameRecorded() {
373 CriticalSectionScoped crit(comparison_lock_.get());
374 ++frames_recorded_;
375 }
376
377 // Returns true if all frames to be compared have been taken from the queue.
378 bool AllFramesRecorded() {
379 CriticalSectionScoped crit(comparison_lock_.get());
380 assert(frames_recorded_ <= frames_to_process_);
381 return frames_recorded_ == frames_to_process_;
382 }
383
384 // Increase count of number of frames processed. Returns true if this was the
385 // last frame to be processed.
386 bool FrameProcessed() {
387 CriticalSectionScoped crit(comparison_lock_.get());
388 ++frames_processed_;
389 assert(frames_processed_ <= frames_to_process_);
390 return frames_processed_ == frames_to_process_;
391 }
392
393 void PrintResults() {
394 CriticalSectionScoped crit(comparison_lock_.get());
395 PrintResult("psnr", psnr_, " dB");
396 PrintResult("ssim", ssim_, "");
397 PrintResult("sender_time", sender_time_, " ms");
398 printf("RESULT dropped_frames: %s = %d frames\n", test_label_,
399 dropped_frames_);
400 PrintResult("receiver_time", receiver_time_, " ms");
401 PrintResult("total_delay_incl_network", end_to_end_, " ms");
402 PrintResult("time_between_rendered_frames", rendered_delta_, " ms");
403 EXPECT_GT(psnr_.Mean(), avg_psnr_threshold_);
404 EXPECT_GT(ssim_.Mean(), avg_ssim_threshold_);
405 }
406
407 void PerformFrameComparison(const FrameComparison& comparison) {
408 // Perform expensive psnr and ssim calculations while not holding lock.
409 double psnr = I420PSNR(&comparison.reference, &comparison.render);
410 double ssim = I420SSIM(&comparison.reference, &comparison.render);
411
412 CriticalSectionScoped crit(comparison_lock_.get());
413 psnr_.AddSample(psnr);
414 ssim_.AddSample(ssim);
415 if (comparison.dropped) {
pbos@webrtc.org94015242013-10-16 11:05:37 +0000416 ++dropped_frames_;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000417 return;
pbos@webrtc.org94015242013-10-16 11:05:37 +0000418 }
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000419 if (last_render_time_ != 0)
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000420 rendered_delta_.AddSample(comparison.render_time_ms - last_render_time_);
421 last_render_time_ = comparison.render_time_ms;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000422
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000423 int64_t input_time_ms = comparison.reference.render_time_ms();
424 sender_time_.AddSample(comparison.send_time_ms - input_time_ms);
425 receiver_time_.AddSample(comparison.render_time_ms -
426 comparison.recv_time_ms);
427 end_to_end_.AddSample(comparison.render_time_ms - input_time_ms);
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000428 }
429
430 void PrintResult(const char* result_type,
431 test::Statistics stats,
432 const char* unit) {
433 printf("RESULT %s: %s = {%f, %f}%s\n",
434 result_type,
435 test_label_,
436 stats.Mean(),
437 stats.StandardDeviation(),
438 unit);
439 }
440
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000441 const char* const test_label_;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000442 test::Statistics sender_time_;
443 test::Statistics receiver_time_;
444 test::Statistics psnr_;
445 test::Statistics ssim_;
446 test::Statistics end_to_end_;
447 test::Statistics rendered_delta_;
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000448 const int frames_to_process_;
449 int frames_recorded_;
450 int frames_processed_;
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000451 int dropped_frames_;
452 int64_t last_render_time_;
453 uint32_t rtp_timestamp_delta_;
454
kwiberg@webrtc.org00b8f6b2015-02-26 14:34:55 +0000455 const rtc::scoped_ptr<CriticalSectionWrapper> crit_;
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000456 std::deque<I420VideoFrame*> frames_ GUARDED_BY(crit_);
457 std::deque<I420VideoFrame*> frame_pool_ GUARDED_BY(crit_);
458 I420VideoFrame last_rendered_frame_ GUARDED_BY(crit_);
459 std::map<uint32_t, int64_t> send_times_ GUARDED_BY(crit_);
460 std::map<uint32_t, int64_t> recv_times_ GUARDED_BY(crit_);
461 I420VideoFrame* first_send_frame_ GUARDED_BY(crit_);
pbos@webrtc.orgb35b1362014-10-20 09:14:38 +0000462 const double avg_psnr_threshold_;
463 const double avg_ssim_threshold_;
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000464
kwiberg@webrtc.org00b8f6b2015-02-26 14:34:55 +0000465 const rtc::scoped_ptr<CriticalSectionWrapper> comparison_lock_;
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000466 std::vector<ThreadWrapper*> comparison_thread_pool_;
kwiberg@webrtc.org00b8f6b2015-02-26 14:34:55 +0000467 const rtc::scoped_ptr<EventWrapper> comparison_available_event_;
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000468 std::deque<FrameComparison> comparisons_ GUARDED_BY(comparison_lock_);
kwiberg@webrtc.org00b8f6b2015-02-26 14:34:55 +0000469 const rtc::scoped_ptr<EventWrapper> done_;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000470};
471
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000472void FullStackTest::RunTest(const FullStackTestParams& params) {
473 test::DirectTransport send_transport(params.link);
474 test::DirectTransport recv_transport(params.link);
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000475 VideoAnalyzer analyzer(NULL, &send_transport, params.test_label,
476 params.avg_psnr_threshold, params.avg_ssim_threshold,
477 params.test_durations_secs * params.clip.fps);
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000478
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000479 CreateCalls(Call::Config(&analyzer), Call::Config(&recv_transport));
mflodman@webrtc.org6879c8a2013-07-23 11:35:00 +0000480
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000481 analyzer.SetReceiver(receiver_call_->Receiver());
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000482 send_transport.SetReceiver(&analyzer);
483 recv_transport.SetReceiver(sender_call_->Receiver());
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000484
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000485 CreateSendConfig(1);
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000486
kwiberg@webrtc.org00b8f6b2015-02-26 14:34:55 +0000487 rtc::scoped_ptr<VideoEncoder> encoder(
pbos@webrtc.orgab990ae2014-09-17 09:02:25 +0000488 VideoEncoder::Create(VideoEncoder::kVp8));
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000489 send_config_.encoder_settings.encoder = encoder.get();
490 send_config_.encoder_settings.payload_name = "VP8";
491 send_config_.encoder_settings.payload_type = 124;
stefan@webrtc.orgc216b9a2014-10-14 10:38:49 +0000492 send_config_.rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000493 send_config_.rtp.rtx.ssrcs.push_back(kSendRtxSsrcs[0]);
494 send_config_.rtp.rtx.payload_type = kSendRtxPayloadType;
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000495
pbos@webrtc.orgbbe0a852014-09-19 12:30:25 +0000496 VideoStream* stream = &encoder_config_.streams[0];
pbos@webrtc.orgf577ae92014-03-19 08:43:57 +0000497 stream->width = params.clip.width;
498 stream->height = params.clip.height;
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000499 stream->min_bitrate_bps = params.min_bitrate_bps;
500 stream->target_bitrate_bps = params.target_bitrate_bps;
501 stream->max_bitrate_bps = params.max_bitrate_bps;
pbos@webrtc.orgf577ae92014-03-19 08:43:57 +0000502 stream->max_framerate = params.clip.fps;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000503
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000504 if (params.screenshare) {
505 encoder_config_.content_type = VideoEncoderConfig::kScreenshare;
506 encoder_config_.min_transmit_bitrate_bps = 400 * 1000;
507 VideoCodecVP8 vp8_settings = VideoEncoder::GetDefaultVp8Settings();
508 vp8_settings.denoisingOn = false;
509 vp8_settings.frameDroppingOn = false;
510 vp8_settings.numberOfTemporalLayers = 2;
511 encoder_config_.encoder_specific_settings = &vp8_settings;
512
513 stream->temporal_layer_thresholds_bps.clear();
514 stream->temporal_layer_thresholds_bps.push_back(stream->target_bitrate_bps);
515 }
516
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000517 CreateMatchingReceiveConfigs();
pbos@webrtc.orgbe9d2a42014-06-30 13:19:09 +0000518 receive_configs_[0].renderer = &analyzer;
stefan@webrtc.orgc216b9a2014-10-14 10:38:49 +0000519 receive_configs_[0].rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000520 receive_configs_[0].rtp.rtx[kSendRtxPayloadType].ssrc = kSendRtxSsrcs[0];
521 receive_configs_[0].rtp.rtx[kSendRtxPayloadType].payload_type =
sprang@webrtc.org343096a2015-02-23 08:34:17 +0000522 kSendRtxPayloadType;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000523
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000524 CreateStreams();
525 analyzer.input_ = send_stream_->Input();
526
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000527 if (params.screenshare) {
528 std::vector<std::string> slides;
529 slides.push_back(test::ResourcePath("web_screenshot_1850_1110", "yuv"));
530 slides.push_back(test::ResourcePath("presentation_1850_1110", "yuv"));
531 slides.push_back(test::ResourcePath("photo_1850_1110", "yuv"));
532 slides.push_back(test::ResourcePath("difficult_photo_1850_1110", "yuv"));
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000533
kwiberg@webrtc.org00b8f6b2015-02-26 14:34:55 +0000534 rtc::scoped_ptr<test::FrameGenerator> frame_generator(
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000535 test::FrameGenerator::CreateFromYuvFile(
536 slides, 1850, 1110,
537 10 * params.clip.fps) // Cycle image every 10 seconds.
538 );
539 frame_generator_capturer_.reset(new test::FrameGeneratorCapturer(
540 Clock::GetRealTimeClock(), &analyzer, frame_generator.release(),
541 params.clip.fps));
542 ASSERT_TRUE(frame_generator_capturer_->Init());
543 } else {
544 frame_generator_capturer_.reset(
545 test::FrameGeneratorCapturer::CreateFromYuvFile(
546 &analyzer, test::ResourcePath(params.clip.name, "yuv"),
547 params.clip.width, params.clip.height, params.clip.fps,
548 Clock::GetRealTimeClock()));
549
550 ASSERT_TRUE(frame_generator_capturer_.get() != NULL)
551 << "Could not create capturer for " << params.clip.name
552 << ".yuv. Is this resource file present?";
553 }
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000554
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000555 Start();
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000556
557 analyzer.Wait();
558
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000559 send_transport.StopSending();
560 recv_transport.StopSending();
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000561
562 Stop();
563
564 DestroyStreams();
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000565}
566
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000567TEST_F(FullStackTest, ParisQcifWithoutPacketLoss) {
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000568 FullStackTestParams paris_qcif = {"net_delay_0_0_plr_0",
569 {"paris_qcif", 176, 144, 30},
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000570 false,
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000571 300000,
572 300000,
573 300000,
574 36.0,
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000575 0.96,
576 kFullStackTestDurationSecs};
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000577 RunTest(paris_qcif);
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000578}
579
580TEST_F(FullStackTest, ForemanCifWithoutPacketLoss) {
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000581 // TODO(pbos): Decide on psnr/ssim thresholds for foreman_cif.
582 FullStackTestParams foreman_cif = {"foreman_cif_net_delay_0_0_plr_0",
583 {"foreman_cif", 352, 288, 30},
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000584 false,
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000585 700000,
586 700000,
587 700000,
588 0.0,
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000589 0.0,
590 kFullStackTestDurationSecs};
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000591 RunTest(foreman_cif);
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000592}
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000593
sprang@webrtc.org343096a2015-02-23 08:34:17 +0000594TEST_F(FullStackTest, ForemanCifPlr5) {
stefan@webrtc.orgc216b9a2014-10-14 10:38:49 +0000595 FullStackTestParams foreman_cif = {"foreman_cif_delay_50_0_plr_5",
596 {"foreman_cif", 352, 288, 30},
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000597 false,
stefan@webrtc.orgc216b9a2014-10-14 10:38:49 +0000598 30000,
599 500000,
600 2000000,
601 0.0,
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000602 0.0,
603 kFullStackTestDurationSecs};
stefan@webrtc.orgc216b9a2014-10-14 10:38:49 +0000604 foreman_cif.link.loss_percent = 5;
605 foreman_cif.link.queue_delay_ms = 50;
606 RunTest(foreman_cif);
607}
608
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000609TEST_F(FullStackTest, ForemanCif500kbps) {
610 FullStackTestParams foreman_cif = {"foreman_cif_500kbps",
611 {"foreman_cif", 352, 288, 30},
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000612 false,
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000613 30000,
614 500000,
615 2000000,
616 0.0,
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000617 0.0,
618 kFullStackTestDurationSecs};
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000619 foreman_cif.link.queue_length_packets = 0;
620 foreman_cif.link.queue_delay_ms = 0;
621 foreman_cif.link.link_capacity_kbps = 500;
622 RunTest(foreman_cif);
623}
624
625TEST_F(FullStackTest, ForemanCif500kbpsLimitedQueue) {
626 FullStackTestParams foreman_cif = {"foreman_cif_500kbps_32pkts_queue",
627 {"foreman_cif", 352, 288, 30},
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000628 false,
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000629 30000,
630 500000,
631 2000000,
632 0.0,
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000633 0.0,
634 kFullStackTestDurationSecs};
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000635 foreman_cif.link.queue_length_packets = 32;
636 foreman_cif.link.queue_delay_ms = 0;
637 foreman_cif.link.link_capacity_kbps = 500;
638 RunTest(foreman_cif);
639}
640
641TEST_F(FullStackTest, ForemanCif500kbps100ms) {
642 FullStackTestParams foreman_cif = {"foreman_cif_500kbps_100ms",
643 {"foreman_cif", 352, 288, 30},
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000644 false,
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000645 30000,
646 500000,
647 2000000,
648 0.0,
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000649 0.0,
650 kFullStackTestDurationSecs};
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000651 foreman_cif.link.queue_length_packets = 0;
652 foreman_cif.link.queue_delay_ms = 100;
653 foreman_cif.link.link_capacity_kbps = 500;
654 RunTest(foreman_cif);
655}
656
657TEST_F(FullStackTest, ForemanCif500kbps100msLimitedQueue) {
658 FullStackTestParams foreman_cif = {"foreman_cif_500kbps_100ms_32pkts_queue",
659 {"foreman_cif", 352, 288, 30},
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000660 false,
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000661 30000,
662 500000,
663 2000000,
664 0.0,
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000665 0.0,
666 kFullStackTestDurationSecs};
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000667 foreman_cif.link.queue_length_packets = 32;
668 foreman_cif.link.queue_delay_ms = 100;
669 foreman_cif.link.link_capacity_kbps = 500;
670 RunTest(foreman_cif);
671}
672
673TEST_F(FullStackTest, ForemanCif1000kbps100msLimitedQueue) {
674 FullStackTestParams foreman_cif = {"foreman_cif_1000kbps_100ms_32pkts_queue",
675 {"foreman_cif", 352, 288, 30},
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000676 false,
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000677 30000,
678 2000000,
679 2000000,
680 0.0,
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000681 0.0,
682 kFullStackTestDurationSecs};
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000683 foreman_cif.link.queue_length_packets = 32;
684 foreman_cif.link.queue_delay_ms = 100;
685 foreman_cif.link.link_capacity_kbps = 1000;
686 RunTest(foreman_cif);
687}
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000688
689TEST_F(FullStackTest, ScreenshareSlides) {
690 FullStackTestParams screenshare_params = {
691 "screenshare_slides",
692 {"screenshare_slides", 1850, 1110, 5},
693 true,
694 50000,
695 100000,
696 1000000,
697 0.0,
698 0.0,
699 kFullStackTestDurationSecs};
700 RunTest(screenshare_params);
701}
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000702} // namespace webrtc