blob: 8ae798a3b9a8ee4ef26f6d3a4dce33b6e563fb7e [file] [log] [blame]
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +00001/*
2 * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10#include <stdio.h>
11
12#include <deque>
13#include <map>
14
pbos@webrtc.org69215d82013-07-10 15:02:02 +000015#include "testing/gtest/include/gtest/gtest.h"
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000016
pbos@webrtc.org38344ed2014-09-24 06:05:00 +000017#include "webrtc/base/thread_annotations.h"
pbos@webrtc.org16e03b72013-10-28 16:32:01 +000018#include "webrtc/call.h"
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000019#include "webrtc/common_video/libyuv/include/webrtc_libyuv.h"
20#include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h"
21#include "webrtc/system_wrappers/interface/clock.h"
sprang@webrtc.org131bea82015-02-18 12:46:06 +000022#include "webrtc/system_wrappers/interface/cpu_info.h"
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000023#include "webrtc/system_wrappers/interface/critical_section_wrapper.h"
24#include "webrtc/system_wrappers/interface/event_wrapper.h"
25#include "webrtc/system_wrappers/interface/scoped_ptr.h"
pbos@webrtc.org94015242013-10-16 11:05:37 +000026#include "webrtc/system_wrappers/interface/sleep.h"
pbos@webrtc.org994d0b72014-06-27 08:47:52 +000027#include "webrtc/test/call_test.h"
pbos@webrtc.org16e03b72013-10-28 16:32:01 +000028#include "webrtc/test/direct_transport.h"
pbos@webrtc.orgf577ae92014-03-19 08:43:57 +000029#include "webrtc/test/encoder_settings.h"
30#include "webrtc/test/fake_encoder.h"
sprang@webrtc.org131bea82015-02-18 12:46:06 +000031#include "webrtc/test/frame_generator.h"
pbos@webrtc.org16e03b72013-10-28 16:32:01 +000032#include "webrtc/test/frame_generator_capturer.h"
pbos@webrtc.org16e03b72013-10-28 16:32:01 +000033#include "webrtc/test/statistics.h"
pbos@webrtc.orgf577ae92014-03-19 08:43:57 +000034#include "webrtc/test/testsupport/fileutils.h"
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000035#include "webrtc/typedefs.h"
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000036
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000037namespace webrtc {
38
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +000039static const int kFullStackTestDurationSecs = 60;
pbos@webrtc.orgb613b5a2013-12-03 10:13:04 +000040
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000041struct FullStackTestParams {
42 const char* test_label;
43 struct {
44 const char* name;
pbos@webrtc.orgf3f13582013-07-09 14:04:46 +000045 size_t width, height;
46 int fps;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000047 } clip;
sprang@webrtc.org131bea82015-02-18 12:46:06 +000048 bool screenshare;
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +000049 int min_bitrate_bps;
50 int target_bitrate_bps;
51 int max_bitrate_bps;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000052 double avg_psnr_threshold;
53 double avg_ssim_threshold;
sprang@webrtc.org131bea82015-02-18 12:46:06 +000054 int test_durations_secs;
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +000055 FakeNetworkPipe::Config link;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000056};
57
pbos@webrtc.org994d0b72014-06-27 08:47:52 +000058class FullStackTest : public test::CallTest {
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000059 protected:
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +000060 void RunTest(const FullStackTestParams& params);
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000061};
62
pbos@webrtc.org74fa4892013-08-23 09:19:30 +000063class VideoAnalyzer : public PacketReceiver,
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000064 public newapi::Transport,
pbos@webrtc.org74fa4892013-08-23 09:19:30 +000065 public VideoRenderer,
66 public VideoSendStreamInput {
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000067 public:
pbos@webrtc.org74fa4892013-08-23 09:19:30 +000068 VideoAnalyzer(VideoSendStreamInput* input,
69 Transport* transport,
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000070 const char* test_label,
71 double avg_psnr_threshold,
72 double avg_ssim_threshold,
pbos@webrtc.org94015242013-10-16 11:05:37 +000073 int duration_frames)
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000074 : input_(input),
75 transport_(transport),
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000076 receiver_(NULL),
77 test_label_(test_label),
sprang@webrtc.org131bea82015-02-18 12:46:06 +000078 frames_to_process_(duration_frames),
79 frames_recorded_(0),
80 frames_processed_(0),
pbos@webrtc.org94015242013-10-16 11:05:37 +000081 dropped_frames_(0),
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000082 last_render_time_(0),
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +000083 rtp_timestamp_delta_(0),
84 crit_(CriticalSectionWrapper::CreateCriticalSection()),
85 first_send_frame_(NULL),
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +000086 avg_psnr_threshold_(avg_psnr_threshold),
87 avg_ssim_threshold_(avg_ssim_threshold),
pbos@webrtc.org94015242013-10-16 11:05:37 +000088 comparison_lock_(CriticalSectionWrapper::CreateCriticalSection()),
sprang@webrtc.org131bea82015-02-18 12:46:06 +000089 comparison_available_event_(EventWrapper::Create()),
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +000090 done_(EventWrapper::Create()) {
sprang@webrtc.org131bea82015-02-18 12:46:06 +000091 // Create thread pool for CPU-expensive PSNR/SSIM calculations.
92
93 // Try to use about as many threads as cores, but leave kMinCoresLeft alone,
94 // so that we don't accidentally starve "real" worker threads (codec etc).
95 // Also, don't allocate more than kMaxComparisonThreads, even if there are
96 // spare cores.
97
98 uint32_t num_cores = CpuInfo::DetectNumberOfCores();
99 assert(num_cores >= 1);
100 static const uint32_t kMinCoresLeft = 4;
101 static const uint32_t kMaxComparisonThreads = 12;
102
103 if (num_cores <= kMinCoresLeft) {
104 num_cores = 1;
105 } else {
106 num_cores -= kMinCoresLeft;
107 num_cores = std::min(num_cores, kMaxComparisonThreads);
108 }
109
110 for (uint32_t i = 0; i < num_cores; ++i) {
111 ThreadWrapper* thread =
112 ThreadWrapper::CreateThread(&FrameComparisonThread, this);
113 comparison_thread_pool_.push_back(thread);
114 unsigned int id;
115 EXPECT_TRUE(thread->Start(id));
116 }
pbos@webrtc.org94015242013-10-16 11:05:37 +0000117 }
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000118
119 ~VideoAnalyzer() {
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000120 for (ThreadWrapper* thread : comparison_thread_pool_) {
121 EXPECT_TRUE(thread->Stop());
122 delete thread;
123 }
pbos@webrtc.org94015242013-10-16 11:05:37 +0000124
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000125 for (I420VideoFrame* frame : frames_)
126 delete frame;
127
128 for (I420VideoFrame* frame : frame_pool_)
129 delete frame;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000130 }
131
pbos@webrtc.org94015242013-10-16 11:05:37 +0000132 virtual void SetReceiver(PacketReceiver* receiver) { receiver_ = receiver; }
133
pbos@webrtc.orgcaba2d22014-05-14 13:57:12 +0000134 virtual DeliveryStatus DeliverPacket(const uint8_t* packet,
135 size_t length) OVERRIDE {
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000136 scoped_ptr<RtpHeaderParser> parser(RtpHeaderParser::Create());
137 RTPHeader header;
pbos@webrtc.org62bafae2014-07-08 12:10:51 +0000138 parser->Parse(packet, length, &header);
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000139 {
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000140 CriticalSectionScoped lock(crit_.get());
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000141 recv_times_[header.timestamp - rtp_timestamp_delta_] =
142 Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
143 }
144
145 return receiver_->DeliverPacket(packet, length);
146 }
147
pbos@webrtc.org724947b2013-12-11 16:26:16 +0000148 virtual void SwapFrame(I420VideoFrame* video_frame) OVERRIDE {
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000149 I420VideoFrame* copy = NULL;
150 {
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000151 CriticalSectionScoped lock(crit_.get());
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000152 if (frame_pool_.size() > 0) {
153 copy = frame_pool_.front();
154 frame_pool_.pop_front();
155 }
156 }
157 if (copy == NULL)
158 copy = new I420VideoFrame();
159
pbos@webrtc.org724947b2013-12-11 16:26:16 +0000160 copy->CopyFrame(*video_frame);
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000161 copy->set_timestamp(copy->render_time_ms() * 90);
162
163 {
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000164 CriticalSectionScoped lock(crit_.get());
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000165 if (first_send_frame_ == NULL && rtp_timestamp_delta_ == 0)
166 first_send_frame_ = copy;
167
168 frames_.push_back(copy);
169 }
170
pbos@webrtc.org724947b2013-12-11 16:26:16 +0000171 input_->SwapFrame(video_frame);
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000172 }
173
pbos@webrtc.org27326b62013-11-20 12:17:04 +0000174 virtual bool SendRtp(const uint8_t* packet, size_t length) OVERRIDE {
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000175 scoped_ptr<RtpHeaderParser> parser(RtpHeaderParser::Create());
176 RTPHeader header;
pbos@webrtc.org62bafae2014-07-08 12:10:51 +0000177 parser->Parse(packet, length, &header);
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000178
179 {
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000180 CriticalSectionScoped lock(crit_.get());
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000181 if (rtp_timestamp_delta_ == 0) {
182 rtp_timestamp_delta_ =
183 header.timestamp - first_send_frame_->timestamp();
184 first_send_frame_ = NULL;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000185 }
pbos@webrtc.org7fb9ce02013-08-05 09:29:50 +0000186 send_times_[header.timestamp - rtp_timestamp_delta_] =
187 Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000188 }
189
pbos@webrtc.org27326b62013-11-20 12:17:04 +0000190 return transport_->SendRtp(packet, length);
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000191 }
192
pbos@webrtc.org27326b62013-11-20 12:17:04 +0000193 virtual bool SendRtcp(const uint8_t* packet, size_t length) OVERRIDE {
194 return transport_->SendRtcp(packet, length);
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000195 }
196
197 virtual void RenderFrame(const I420VideoFrame& video_frame,
198 int time_to_render_ms) OVERRIDE {
pbos@webrtc.org94015242013-10-16 11:05:37 +0000199 int64_t render_time_ms =
200 Clock::GetRealTimeClock()->CurrentNtpInMilliseconds();
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000201 uint32_t send_timestamp = video_frame.timestamp() - rtp_timestamp_delta_;
202
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000203 CriticalSectionScoped lock(crit_.get());
204 while (frames_.front()->timestamp() < send_timestamp) {
205 AddFrameComparison(
206 frames_.front(), &last_rendered_frame_, true, render_time_ms);
207 frame_pool_.push_back(frames_.front());
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000208 frames_.pop_front();
pbos@webrtc.org94015242013-10-16 11:05:37 +0000209 }
210
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000211 I420VideoFrame* reference_frame = frames_.front();
212 frames_.pop_front();
213 assert(reference_frame != NULL);
214 EXPECT_EQ(reference_frame->timestamp(), send_timestamp);
215 assert(reference_frame->timestamp() == send_timestamp);
216
217 AddFrameComparison(reference_frame, &video_frame, false, render_time_ms);
218 frame_pool_.push_back(reference_frame);
219
pbos@webrtc.org94015242013-10-16 11:05:37 +0000220 last_rendered_frame_.CopyFrame(video_frame);
221 }
222
pbos@webrtc.org0d852d52015-02-09 15:14:36 +0000223 virtual bool IsTextureSupported() const override { return false; }
224
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000225 void Wait() {
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000226 // Frame comparisons can be very expensive. Wait for test to be done, but
227 // at time-out check if frames_processed is going up. If so, give it more
228 // time, otherwise fail. Hopefully this will reduce test flakiness.
229
230 int last_frames_processed = -1;
231 EventTypeWrapper eventType;
232 while ((eventType = done_->Wait(FullStackTest::kDefaultTimeoutMs)) !=
233 kEventSignaled) {
234 int frames_processed;
235 {
236 CriticalSectionScoped crit(comparison_lock_.get());
237 frames_processed = frames_processed_;
238 }
239 if (last_frames_processed == -1) {
240 last_frames_processed = frames_processed;
241 continue;
242 }
243 ASSERT_GT(frames_processed, last_frames_processed)
244 << "Analyzer stalled while waiting for test to finish.";
245 last_frames_processed = frames_processed;
246 }
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000247 }
pbos@webrtc.org94015242013-10-16 11:05:37 +0000248
249 VideoSendStreamInput* input_;
250 Transport* transport_;
251 PacketReceiver* receiver_;
252
253 private:
254 struct FrameComparison {
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000255 FrameComparison()
256 : dropped(false), send_time_ms(0), recv_time_ms(0), render_time_ms(0) {}
257
pbos@webrtc.org94015242013-10-16 11:05:37 +0000258 FrameComparison(const I420VideoFrame* reference,
259 const I420VideoFrame* render,
260 bool dropped,
261 int64_t send_time_ms,
262 int64_t recv_time_ms,
263 int64_t render_time_ms)
264 : dropped(dropped),
265 send_time_ms(send_time_ms),
266 recv_time_ms(recv_time_ms),
267 render_time_ms(render_time_ms) {
268 this->reference.CopyFrame(*reference);
269 this->render.CopyFrame(*render);
270 }
271
272 FrameComparison(const FrameComparison& compare)
273 : dropped(compare.dropped),
274 send_time_ms(compare.send_time_ms),
275 recv_time_ms(compare.recv_time_ms),
276 render_time_ms(compare.render_time_ms) {
277 this->reference.CopyFrame(compare.reference);
278 this->render.CopyFrame(compare.render);
279 }
280
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000281 void CloneStatsAndSwapFrames(FrameComparison* comparison) {
282 reference.SwapFrame(&comparison->reference);
283 render.SwapFrame(&comparison->render);
284 dropped = comparison->dropped;
285 send_time_ms = comparison->send_time_ms;
286 recv_time_ms = comparison->recv_time_ms;
287 render_time_ms = comparison->render_time_ms;
288 }
289
pbos@webrtc.org94015242013-10-16 11:05:37 +0000290 ~FrameComparison() {}
291
292 I420VideoFrame reference;
293 I420VideoFrame render;
294 bool dropped;
295 int64_t send_time_ms;
296 int64_t recv_time_ms;
297 int64_t render_time_ms;
298 };
299
300 void AddFrameComparison(const I420VideoFrame* reference,
301 const I420VideoFrame* render,
302 bool dropped,
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000303 int64_t render_time_ms)
304 EXCLUSIVE_LOCKS_REQUIRED(crit_) {
pbos@webrtc.org94015242013-10-16 11:05:37 +0000305 int64_t send_time_ms = send_times_[reference->timestamp()];
306 send_times_.erase(reference->timestamp());
307 int64_t recv_time_ms = recv_times_[reference->timestamp()];
308 recv_times_.erase(reference->timestamp());
309
310 CriticalSectionScoped crit(comparison_lock_.get());
311 comparisons_.push_back(FrameComparison(reference,
312 render,
313 dropped,
314 send_time_ms,
315 recv_time_ms,
316 render_time_ms));
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000317 comparison_available_event_->Set();
pbos@webrtc.org94015242013-10-16 11:05:37 +0000318 }
319
320 static bool FrameComparisonThread(void* obj) {
321 return static_cast<VideoAnalyzer*>(obj)->CompareFrames();
322 }
323
324 bool CompareFrames() {
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000325 if (AllFramesRecorded())
326 return false;
pbos@webrtc.org94015242013-10-16 11:05:37 +0000327
328 I420VideoFrame reference;
329 I420VideoFrame render;
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000330 FrameComparison comparison;
pbos@webrtc.org94015242013-10-16 11:05:37 +0000331
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000332 if (!PopComparison(&comparison)) {
333 // Wait until new comparison task is available, or test is done.
334 // If done, wake up remaining threads waiting.
335 comparison_available_event_->Wait(1000);
336 if (AllFramesRecorded()) {
337 comparison_available_event_->Set();
pbos@webrtc.org94015242013-10-16 11:05:37 +0000338 return false;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000339 }
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000340 return true; // Try again.
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000341 }
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000342
343 PerformFrameComparison(comparison);
344
345 if (FrameProcessed()) {
346 PrintResults();
347 done_->Set();
348 comparison_available_event_->Set();
349 return false;
350 }
351
352 return true;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000353 }
354
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000355 bool PopComparison(FrameComparison* comparison) {
356 CriticalSectionScoped crit(comparison_lock_.get());
357 // If AllFramesRecorded() is true, it means we have already popped
358 // frames_to_process_ frames from comparisons_, so there is no more work
359 // for this thread to be done. frames_processed_ might still be lower if
360 // all comparisons are not done, but those frames are currently being
361 // worked on by other threads.
362 if (comparisons_.empty() || AllFramesRecorded())
363 return false;
364
365 comparison->CloneStatsAndSwapFrames(&comparisons_.front());
366 comparisons_.pop_front();
367
368 FrameRecorded();
369 return true;
370 }
371
372 // Increment counter for number of frames received for comparison.
373 void FrameRecorded() {
374 CriticalSectionScoped crit(comparison_lock_.get());
375 ++frames_recorded_;
376 }
377
378 // Returns true if all frames to be compared have been taken from the queue.
379 bool AllFramesRecorded() {
380 CriticalSectionScoped crit(comparison_lock_.get());
381 assert(frames_recorded_ <= frames_to_process_);
382 return frames_recorded_ == frames_to_process_;
383 }
384
385 // Increase count of number of frames processed. Returns true if this was the
386 // last frame to be processed.
387 bool FrameProcessed() {
388 CriticalSectionScoped crit(comparison_lock_.get());
389 ++frames_processed_;
390 assert(frames_processed_ <= frames_to_process_);
391 return frames_processed_ == frames_to_process_;
392 }
393
394 void PrintResults() {
395 CriticalSectionScoped crit(comparison_lock_.get());
396 PrintResult("psnr", psnr_, " dB");
397 PrintResult("ssim", ssim_, "");
398 PrintResult("sender_time", sender_time_, " ms");
399 printf("RESULT dropped_frames: %s = %d frames\n", test_label_,
400 dropped_frames_);
401 PrintResult("receiver_time", receiver_time_, " ms");
402 PrintResult("total_delay_incl_network", end_to_end_, " ms");
403 PrintResult("time_between_rendered_frames", rendered_delta_, " ms");
404 EXPECT_GT(psnr_.Mean(), avg_psnr_threshold_);
405 EXPECT_GT(ssim_.Mean(), avg_ssim_threshold_);
406 }
407
408 void PerformFrameComparison(const FrameComparison& comparison) {
409 // Perform expensive psnr and ssim calculations while not holding lock.
410 double psnr = I420PSNR(&comparison.reference, &comparison.render);
411 double ssim = I420SSIM(&comparison.reference, &comparison.render);
412
413 CriticalSectionScoped crit(comparison_lock_.get());
414 psnr_.AddSample(psnr);
415 ssim_.AddSample(ssim);
416 if (comparison.dropped) {
pbos@webrtc.org94015242013-10-16 11:05:37 +0000417 ++dropped_frames_;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000418 return;
pbos@webrtc.org94015242013-10-16 11:05:37 +0000419 }
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000420 if (last_render_time_ != 0)
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000421 rendered_delta_.AddSample(comparison.render_time_ms - last_render_time_);
422 last_render_time_ = comparison.render_time_ms;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000423
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000424 int64_t input_time_ms = comparison.reference.render_time_ms();
425 sender_time_.AddSample(comparison.send_time_ms - input_time_ms);
426 receiver_time_.AddSample(comparison.render_time_ms -
427 comparison.recv_time_ms);
428 end_to_end_.AddSample(comparison.render_time_ms - input_time_ms);
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000429 }
430
431 void PrintResult(const char* result_type,
432 test::Statistics stats,
433 const char* unit) {
434 printf("RESULT %s: %s = {%f, %f}%s\n",
435 result_type,
436 test_label_,
437 stats.Mean(),
438 stats.StandardDeviation(),
439 unit);
440 }
441
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000442 const char* const test_label_;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000443 test::Statistics sender_time_;
444 test::Statistics receiver_time_;
445 test::Statistics psnr_;
446 test::Statistics ssim_;
447 test::Statistics end_to_end_;
448 test::Statistics rendered_delta_;
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000449 const int frames_to_process_;
450 int frames_recorded_;
451 int frames_processed_;
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000452 int dropped_frames_;
453 int64_t last_render_time_;
454 uint32_t rtp_timestamp_delta_;
455
456 const scoped_ptr<CriticalSectionWrapper> crit_;
457 std::deque<I420VideoFrame*> frames_ GUARDED_BY(crit_);
458 std::deque<I420VideoFrame*> frame_pool_ GUARDED_BY(crit_);
459 I420VideoFrame last_rendered_frame_ GUARDED_BY(crit_);
460 std::map<uint32_t, int64_t> send_times_ GUARDED_BY(crit_);
461 std::map<uint32_t, int64_t> recv_times_ GUARDED_BY(crit_);
462 I420VideoFrame* first_send_frame_ GUARDED_BY(crit_);
pbos@webrtc.orgb35b1362014-10-20 09:14:38 +0000463 const double avg_psnr_threshold_;
464 const double avg_ssim_threshold_;
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000465
466 const scoped_ptr<CriticalSectionWrapper> comparison_lock_;
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000467 std::vector<ThreadWrapper*> comparison_thread_pool_;
468 const scoped_ptr<EventWrapper> comparison_available_event_;
pbos@webrtc.orgde1429e2014-04-28 13:00:21 +0000469 std::deque<FrameComparison> comparisons_ GUARDED_BY(comparison_lock_);
470 const scoped_ptr<EventWrapper> done_;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000471};
472
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000473void FullStackTest::RunTest(const FullStackTestParams& params) {
474 test::DirectTransport send_transport(params.link);
475 test::DirectTransport recv_transport(params.link);
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000476 VideoAnalyzer analyzer(NULL, &send_transport, params.test_label,
477 params.avg_psnr_threshold, params.avg_ssim_threshold,
478 params.test_durations_secs * params.clip.fps);
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000479
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000480 CreateCalls(Call::Config(&analyzer), Call::Config(&recv_transport));
mflodman@webrtc.org6879c8a2013-07-23 11:35:00 +0000481
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000482 analyzer.SetReceiver(receiver_call_->Receiver());
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000483 send_transport.SetReceiver(&analyzer);
484 recv_transport.SetReceiver(sender_call_->Receiver());
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000485
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000486 CreateSendConfig(1);
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000487
pbos@webrtc.orgab990ae2014-09-17 09:02:25 +0000488 scoped_ptr<VideoEncoder> encoder(
489 VideoEncoder::Create(VideoEncoder::kVp8));
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000490 send_config_.encoder_settings.encoder = encoder.get();
491 send_config_.encoder_settings.payload_name = "VP8";
492 send_config_.encoder_settings.payload_type = 124;
stefan@webrtc.orgc216b9a2014-10-14 10:38:49 +0000493 send_config_.rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000494 send_config_.rtp.rtx.ssrcs.push_back(kSendRtxSsrcs[0]);
495 send_config_.rtp.rtx.payload_type = kSendRtxPayloadType;
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000496
pbos@webrtc.orgbbe0a852014-09-19 12:30:25 +0000497 VideoStream* stream = &encoder_config_.streams[0];
pbos@webrtc.orgf577ae92014-03-19 08:43:57 +0000498 stream->width = params.clip.width;
499 stream->height = params.clip.height;
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000500 stream->min_bitrate_bps = params.min_bitrate_bps;
501 stream->target_bitrate_bps = params.target_bitrate_bps;
502 stream->max_bitrate_bps = params.max_bitrate_bps;
pbos@webrtc.orgf577ae92014-03-19 08:43:57 +0000503 stream->max_framerate = params.clip.fps;
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000504
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000505 if (params.screenshare) {
506 encoder_config_.content_type = VideoEncoderConfig::kScreenshare;
507 encoder_config_.min_transmit_bitrate_bps = 400 * 1000;
508 VideoCodecVP8 vp8_settings = VideoEncoder::GetDefaultVp8Settings();
509 vp8_settings.denoisingOn = false;
510 vp8_settings.frameDroppingOn = false;
511 vp8_settings.numberOfTemporalLayers = 2;
512 encoder_config_.encoder_specific_settings = &vp8_settings;
513
514 stream->temporal_layer_thresholds_bps.clear();
515 stream->temporal_layer_thresholds_bps.push_back(stream->target_bitrate_bps);
516 }
517
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000518 CreateMatchingReceiveConfigs();
pbos@webrtc.orgbe9d2a42014-06-30 13:19:09 +0000519 receive_configs_[0].renderer = &analyzer;
stefan@webrtc.orgc216b9a2014-10-14 10:38:49 +0000520 receive_configs_[0].rtp.nack.rtp_history_ms = kNackRtpHistoryMs;
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000521 receive_configs_[0].rtp.rtx[kSendRtxPayloadType].ssrc = kSendRtxSsrcs[0];
522 receive_configs_[0].rtp.rtx[kSendRtxPayloadType].payload_type =
523 kSendRtxSsrcs[0];
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000524
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000525 CreateStreams();
526 analyzer.input_ = send_stream_->Input();
527
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000528 if (params.screenshare) {
529 std::vector<std::string> slides;
530 slides.push_back(test::ResourcePath("web_screenshot_1850_1110", "yuv"));
531 slides.push_back(test::ResourcePath("presentation_1850_1110", "yuv"));
532 slides.push_back(test::ResourcePath("photo_1850_1110", "yuv"));
533 slides.push_back(test::ResourcePath("difficult_photo_1850_1110", "yuv"));
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000534
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000535 scoped_ptr<test::FrameGenerator> frame_generator(
536 test::FrameGenerator::CreateFromYuvFile(
537 slides, 1850, 1110,
538 10 * params.clip.fps) // Cycle image every 10 seconds.
539 );
540 frame_generator_capturer_.reset(new test::FrameGeneratorCapturer(
541 Clock::GetRealTimeClock(), &analyzer, frame_generator.release(),
542 params.clip.fps));
543 ASSERT_TRUE(frame_generator_capturer_->Init());
544 } else {
545 frame_generator_capturer_.reset(
546 test::FrameGeneratorCapturer::CreateFromYuvFile(
547 &analyzer, test::ResourcePath(params.clip.name, "yuv"),
548 params.clip.width, params.clip.height, params.clip.fps,
549 Clock::GetRealTimeClock()));
550
551 ASSERT_TRUE(frame_generator_capturer_.get() != NULL)
552 << "Could not create capturer for " << params.clip.name
553 << ".yuv. Is this resource file present?";
554 }
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000555
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000556 Start();
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000557
558 analyzer.Wait();
559
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000560 send_transport.StopSending();
561 recv_transport.StopSending();
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000562
563 Stop();
564
565 DestroyStreams();
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000566}
567
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000568TEST_F(FullStackTest, ParisQcifWithoutPacketLoss) {
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000569 FullStackTestParams paris_qcif = {"net_delay_0_0_plr_0",
570 {"paris_qcif", 176, 144, 30},
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000571 false,
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000572 300000,
573 300000,
574 300000,
575 36.0,
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000576 0.96,
577 kFullStackTestDurationSecs};
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000578 RunTest(paris_qcif);
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000579}
580
581TEST_F(FullStackTest, ForemanCifWithoutPacketLoss) {
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000582 // TODO(pbos): Decide on psnr/ssim thresholds for foreman_cif.
583 FullStackTestParams foreman_cif = {"foreman_cif_net_delay_0_0_plr_0",
584 {"foreman_cif", 352, 288, 30},
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000585 false,
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000586 700000,
587 700000,
588 700000,
589 0.0,
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000590 0.0,
591 kFullStackTestDurationSecs};
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000592 RunTest(foreman_cif);
pbos@webrtc.org994d0b72014-06-27 08:47:52 +0000593}
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000594
stefan@webrtc.orgc216b9a2014-10-14 10:38:49 +0000595TEST_F(FullStackTest, ForemanCifPlr5) {
596 FullStackTestParams foreman_cif = {"foreman_cif_delay_50_0_plr_5",
597 {"foreman_cif", 352, 288, 30},
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000598 false,
stefan@webrtc.orgc216b9a2014-10-14 10:38:49 +0000599 30000,
600 500000,
601 2000000,
602 0.0,
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000603 0.0,
604 kFullStackTestDurationSecs};
stefan@webrtc.orgc216b9a2014-10-14 10:38:49 +0000605 foreman_cif.link.loss_percent = 5;
606 foreman_cif.link.queue_delay_ms = 50;
607 RunTest(foreman_cif);
608}
609
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000610TEST_F(FullStackTest, ForemanCif500kbps) {
611 FullStackTestParams foreman_cif = {"foreman_cif_500kbps",
612 {"foreman_cif", 352, 288, 30},
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000613 false,
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000614 30000,
615 500000,
616 2000000,
617 0.0,
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000618 0.0,
619 kFullStackTestDurationSecs};
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000620 foreman_cif.link.queue_length_packets = 0;
621 foreman_cif.link.queue_delay_ms = 0;
622 foreman_cif.link.link_capacity_kbps = 500;
623 RunTest(foreman_cif);
624}
625
626TEST_F(FullStackTest, ForemanCif500kbpsLimitedQueue) {
627 FullStackTestParams foreman_cif = {"foreman_cif_500kbps_32pkts_queue",
628 {"foreman_cif", 352, 288, 30},
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000629 false,
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000630 30000,
631 500000,
632 2000000,
633 0.0,
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000634 0.0,
635 kFullStackTestDurationSecs};
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000636 foreman_cif.link.queue_length_packets = 32;
637 foreman_cif.link.queue_delay_ms = 0;
638 foreman_cif.link.link_capacity_kbps = 500;
639 RunTest(foreman_cif);
640}
641
642TEST_F(FullStackTest, ForemanCif500kbps100ms) {
643 FullStackTestParams foreman_cif = {"foreman_cif_500kbps_100ms",
644 {"foreman_cif", 352, 288, 30},
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000645 false,
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000646 30000,
647 500000,
648 2000000,
649 0.0,
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000650 0.0,
651 kFullStackTestDurationSecs};
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000652 foreman_cif.link.queue_length_packets = 0;
653 foreman_cif.link.queue_delay_ms = 100;
654 foreman_cif.link.link_capacity_kbps = 500;
655 RunTest(foreman_cif);
656}
657
658TEST_F(FullStackTest, ForemanCif500kbps100msLimitedQueue) {
659 FullStackTestParams foreman_cif = {"foreman_cif_500kbps_100ms_32pkts_queue",
660 {"foreman_cif", 352, 288, 30},
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000661 false,
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000662 30000,
663 500000,
664 2000000,
665 0.0,
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000666 0.0,
667 kFullStackTestDurationSecs};
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000668 foreman_cif.link.queue_length_packets = 32;
669 foreman_cif.link.queue_delay_ms = 100;
670 foreman_cif.link.link_capacity_kbps = 500;
671 RunTest(foreman_cif);
672}
673
674TEST_F(FullStackTest, ForemanCif1000kbps100msLimitedQueue) {
675 FullStackTestParams foreman_cif = {"foreman_cif_1000kbps_100ms_32pkts_queue",
676 {"foreman_cif", 352, 288, 30},
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000677 false,
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000678 30000,
679 2000000,
680 2000000,
681 0.0,
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000682 0.0,
683 kFullStackTestDurationSecs};
stefan@webrtc.orgb8e9e442014-07-09 11:29:06 +0000684 foreman_cif.link.queue_length_packets = 32;
685 foreman_cif.link.queue_delay_ms = 100;
686 foreman_cif.link.link_capacity_kbps = 1000;
687 RunTest(foreman_cif);
688}
sprang@webrtc.org131bea82015-02-18 12:46:06 +0000689
690TEST_F(FullStackTest, ScreenshareSlides) {
691 FullStackTestParams screenshare_params = {
692 "screenshare_slides",
693 {"screenshare_slides", 1850, 1110, 5},
694 true,
695 50000,
696 100000,
697 1000000,
698 0.0,
699 0.0,
700 kFullStackTestDurationSecs};
701 RunTest(screenshare_params);
702}
pbos@webrtc.orgaf8d5af2013-07-09 08:02:33 +0000703} // namespace webrtc