pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 1 | /* |
| 2 | * Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | #include <stdio.h> |
| 11 | |
| 12 | #include <deque> |
| 13 | #include <map> |
| 14 | |
pbos@webrtc.org | 69215d8 | 2013-07-10 15:02:02 +0000 | [diff] [blame] | 15 | #include "testing/gtest/include/gtest/gtest.h" |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 16 | |
pbos@webrtc.org | 16e03b7 | 2013-10-28 16:32:01 +0000 | [diff] [blame] | 17 | #include "webrtc/call.h" |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 18 | #include "webrtc/common_video/libyuv/include/webrtc_libyuv.h" |
| 19 | #include "webrtc/modules/rtp_rtcp/interface/rtp_header_parser.h" |
pbos@webrtc.org | f577ae9 | 2014-03-19 08:43:57 +0000 | [diff] [blame] | 20 | #include "webrtc/modules/video_coding/codecs/vp8/include/vp8.h" |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 21 | #include "webrtc/system_wrappers/interface/clock.h" |
| 22 | #include "webrtc/system_wrappers/interface/critical_section_wrapper.h" |
| 23 | #include "webrtc/system_wrappers/interface/event_wrapper.h" |
| 24 | #include "webrtc/system_wrappers/interface/scoped_ptr.h" |
pbos@webrtc.org | 9401524 | 2013-10-16 11:05:37 +0000 | [diff] [blame] | 25 | #include "webrtc/system_wrappers/interface/sleep.h" |
pbos@webrtc.org | de1429e | 2014-04-28 13:00:21 +0000 | [diff] [blame] | 26 | #include "webrtc/system_wrappers/interface/thread_annotations.h" |
pbos@webrtc.org | 994d0b7 | 2014-06-27 08:47:52 +0000 | [diff] [blame] | 27 | #include "webrtc/test/call_test.h" |
pbos@webrtc.org | 16e03b7 | 2013-10-28 16:32:01 +0000 | [diff] [blame] | 28 | #include "webrtc/test/direct_transport.h" |
pbos@webrtc.org | f577ae9 | 2014-03-19 08:43:57 +0000 | [diff] [blame] | 29 | #include "webrtc/test/encoder_settings.h" |
| 30 | #include "webrtc/test/fake_encoder.h" |
pbos@webrtc.org | 16e03b7 | 2013-10-28 16:32:01 +0000 | [diff] [blame] | 31 | #include "webrtc/test/frame_generator_capturer.h" |
pbos@webrtc.org | 16e03b7 | 2013-10-28 16:32:01 +0000 | [diff] [blame] | 32 | #include "webrtc/test/statistics.h" |
pbos@webrtc.org | f577ae9 | 2014-03-19 08:43:57 +0000 | [diff] [blame] | 33 | #include "webrtc/test/testsupport/fileutils.h" |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 34 | #include "webrtc/typedefs.h" |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 35 | |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 36 | namespace webrtc { |
| 37 | |
pbos@webrtc.org | 023b101 | 2014-05-13 11:26:40 +0000 | [diff] [blame] | 38 | static const int kFullStackTestDurationSecs = 10; |
pbos@webrtc.org | b613b5a | 2013-12-03 10:13:04 +0000 | [diff] [blame] | 39 | |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 40 | struct FullStackTestParams { |
| 41 | const char* test_label; |
| 42 | struct { |
| 43 | const char* name; |
pbos@webrtc.org | f3f1358 | 2013-07-09 14:04:46 +0000 | [diff] [blame] | 44 | size_t width, height; |
| 45 | int fps; |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 46 | } clip; |
pbos@webrtc.org | f3f1358 | 2013-07-09 14:04:46 +0000 | [diff] [blame] | 47 | unsigned int bitrate; |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 48 | double avg_psnr_threshold; |
| 49 | double avg_ssim_threshold; |
| 50 | }; |
| 51 | |
pbos@webrtc.org | 994d0b7 | 2014-06-27 08:47:52 +0000 | [diff] [blame] | 52 | class FullStackTest : public test::CallTest { |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 53 | protected: |
pbos@webrtc.org | 994d0b7 | 2014-06-27 08:47:52 +0000 | [diff] [blame] | 54 | void TestWithoutPacketLoss(const FullStackTestParams& params); |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 55 | }; |
| 56 | |
pbos@webrtc.org | 74fa489 | 2013-08-23 09:19:30 +0000 | [diff] [blame] | 57 | class VideoAnalyzer : public PacketReceiver, |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 58 | public newapi::Transport, |
pbos@webrtc.org | 74fa489 | 2013-08-23 09:19:30 +0000 | [diff] [blame] | 59 | public VideoRenderer, |
| 60 | public VideoSendStreamInput { |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 61 | public: |
pbos@webrtc.org | 74fa489 | 2013-08-23 09:19:30 +0000 | [diff] [blame] | 62 | VideoAnalyzer(VideoSendStreamInput* input, |
| 63 | Transport* transport, |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 64 | const char* test_label, |
| 65 | double avg_psnr_threshold, |
| 66 | double avg_ssim_threshold, |
pbos@webrtc.org | 9401524 | 2013-10-16 11:05:37 +0000 | [diff] [blame] | 67 | int duration_frames) |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 68 | : input_(input), |
| 69 | transport_(transport), |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 70 | receiver_(NULL), |
| 71 | test_label_(test_label), |
pbos@webrtc.org | de1429e | 2014-04-28 13:00:21 +0000 | [diff] [blame] | 72 | frames_left_(duration_frames), |
pbos@webrtc.org | 9401524 | 2013-10-16 11:05:37 +0000 | [diff] [blame] | 73 | dropped_frames_(0), |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 74 | last_render_time_(0), |
pbos@webrtc.org | de1429e | 2014-04-28 13:00:21 +0000 | [diff] [blame] | 75 | rtp_timestamp_delta_(0), |
| 76 | crit_(CriticalSectionWrapper::CreateCriticalSection()), |
| 77 | first_send_frame_(NULL), |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 78 | avg_psnr_threshold_(avg_psnr_threshold), |
| 79 | avg_ssim_threshold_(avg_ssim_threshold), |
pbos@webrtc.org | 9401524 | 2013-10-16 11:05:37 +0000 | [diff] [blame] | 80 | comparison_lock_(CriticalSectionWrapper::CreateCriticalSection()), |
| 81 | comparison_thread_(ThreadWrapper::CreateThread(&FrameComparisonThread, |
| 82 | this)), |
pbos@webrtc.org | de1429e | 2014-04-28 13:00:21 +0000 | [diff] [blame] | 83 | done_(EventWrapper::Create()) { |
pbos@webrtc.org | 9401524 | 2013-10-16 11:05:37 +0000 | [diff] [blame] | 84 | unsigned int id; |
| 85 | EXPECT_TRUE(comparison_thread_->Start(id)); |
| 86 | } |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 87 | |
| 88 | ~VideoAnalyzer() { |
pbos@webrtc.org | 9401524 | 2013-10-16 11:05:37 +0000 | [diff] [blame] | 89 | EXPECT_TRUE(comparison_thread_->Stop()); |
| 90 | |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 91 | while (!frames_.empty()) { |
| 92 | delete frames_.back(); |
| 93 | frames_.pop_back(); |
| 94 | } |
| 95 | while (!frame_pool_.empty()) { |
| 96 | delete frame_pool_.back(); |
| 97 | frame_pool_.pop_back(); |
| 98 | } |
| 99 | } |
| 100 | |
pbos@webrtc.org | 9401524 | 2013-10-16 11:05:37 +0000 | [diff] [blame] | 101 | virtual void SetReceiver(PacketReceiver* receiver) { receiver_ = receiver; } |
| 102 | |
pbos@webrtc.org | caba2d2 | 2014-05-14 13:57:12 +0000 | [diff] [blame] | 103 | virtual DeliveryStatus DeliverPacket(const uint8_t* packet, |
| 104 | size_t length) OVERRIDE { |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 105 | scoped_ptr<RtpHeaderParser> parser(RtpHeaderParser::Create()); |
| 106 | RTPHeader header; |
pbos@webrtc.org | 62bafae | 2014-07-08 12:10:51 +0000 | [diff] [blame] | 107 | parser->Parse(packet, length, &header); |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 108 | { |
pbos@webrtc.org | de1429e | 2014-04-28 13:00:21 +0000 | [diff] [blame] | 109 | CriticalSectionScoped lock(crit_.get()); |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 110 | recv_times_[header.timestamp - rtp_timestamp_delta_] = |
| 111 | Clock::GetRealTimeClock()->CurrentNtpInMilliseconds(); |
| 112 | } |
| 113 | |
| 114 | return receiver_->DeliverPacket(packet, length); |
| 115 | } |
| 116 | |
pbos@webrtc.org | 724947b | 2013-12-11 16:26:16 +0000 | [diff] [blame] | 117 | virtual void SwapFrame(I420VideoFrame* video_frame) OVERRIDE { |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 118 | I420VideoFrame* copy = NULL; |
| 119 | { |
pbos@webrtc.org | de1429e | 2014-04-28 13:00:21 +0000 | [diff] [blame] | 120 | CriticalSectionScoped lock(crit_.get()); |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 121 | if (frame_pool_.size() > 0) { |
| 122 | copy = frame_pool_.front(); |
| 123 | frame_pool_.pop_front(); |
| 124 | } |
| 125 | } |
| 126 | if (copy == NULL) |
| 127 | copy = new I420VideoFrame(); |
| 128 | |
pbos@webrtc.org | 724947b | 2013-12-11 16:26:16 +0000 | [diff] [blame] | 129 | copy->CopyFrame(*video_frame); |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 130 | copy->set_timestamp(copy->render_time_ms() * 90); |
| 131 | |
| 132 | { |
pbos@webrtc.org | de1429e | 2014-04-28 13:00:21 +0000 | [diff] [blame] | 133 | CriticalSectionScoped lock(crit_.get()); |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 134 | if (first_send_frame_ == NULL && rtp_timestamp_delta_ == 0) |
| 135 | first_send_frame_ = copy; |
| 136 | |
| 137 | frames_.push_back(copy); |
| 138 | } |
| 139 | |
pbos@webrtc.org | 724947b | 2013-12-11 16:26:16 +0000 | [diff] [blame] | 140 | input_->SwapFrame(video_frame); |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 141 | } |
| 142 | |
pbos@webrtc.org | 27326b6 | 2013-11-20 12:17:04 +0000 | [diff] [blame] | 143 | virtual bool SendRtp(const uint8_t* packet, size_t length) OVERRIDE { |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 144 | scoped_ptr<RtpHeaderParser> parser(RtpHeaderParser::Create()); |
| 145 | RTPHeader header; |
pbos@webrtc.org | 62bafae | 2014-07-08 12:10:51 +0000 | [diff] [blame] | 146 | parser->Parse(packet, length, &header); |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 147 | |
| 148 | { |
pbos@webrtc.org | de1429e | 2014-04-28 13:00:21 +0000 | [diff] [blame] | 149 | CriticalSectionScoped lock(crit_.get()); |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 150 | if (rtp_timestamp_delta_ == 0) { |
| 151 | rtp_timestamp_delta_ = |
| 152 | header.timestamp - first_send_frame_->timestamp(); |
| 153 | first_send_frame_ = NULL; |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 154 | } |
pbos@webrtc.org | 7fb9ce0 | 2013-08-05 09:29:50 +0000 | [diff] [blame] | 155 | send_times_[header.timestamp - rtp_timestamp_delta_] = |
| 156 | Clock::GetRealTimeClock()->CurrentNtpInMilliseconds(); |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 157 | } |
| 158 | |
pbos@webrtc.org | 27326b6 | 2013-11-20 12:17:04 +0000 | [diff] [blame] | 159 | return transport_->SendRtp(packet, length); |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 160 | } |
| 161 | |
pbos@webrtc.org | 27326b6 | 2013-11-20 12:17:04 +0000 | [diff] [blame] | 162 | virtual bool SendRtcp(const uint8_t* packet, size_t length) OVERRIDE { |
| 163 | return transport_->SendRtcp(packet, length); |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 164 | } |
| 165 | |
| 166 | virtual void RenderFrame(const I420VideoFrame& video_frame, |
| 167 | int time_to_render_ms) OVERRIDE { |
pbos@webrtc.org | 9401524 | 2013-10-16 11:05:37 +0000 | [diff] [blame] | 168 | int64_t render_time_ms = |
| 169 | Clock::GetRealTimeClock()->CurrentNtpInMilliseconds(); |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 170 | uint32_t send_timestamp = video_frame.timestamp() - rtp_timestamp_delta_; |
| 171 | |
pbos@webrtc.org | de1429e | 2014-04-28 13:00:21 +0000 | [diff] [blame] | 172 | CriticalSectionScoped lock(crit_.get()); |
| 173 | while (frames_.front()->timestamp() < send_timestamp) { |
| 174 | AddFrameComparison( |
| 175 | frames_.front(), &last_rendered_frame_, true, render_time_ms); |
| 176 | frame_pool_.push_back(frames_.front()); |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 177 | frames_.pop_front(); |
pbos@webrtc.org | 9401524 | 2013-10-16 11:05:37 +0000 | [diff] [blame] | 178 | } |
| 179 | |
pbos@webrtc.org | de1429e | 2014-04-28 13:00:21 +0000 | [diff] [blame] | 180 | I420VideoFrame* reference_frame = frames_.front(); |
| 181 | frames_.pop_front(); |
| 182 | assert(reference_frame != NULL); |
| 183 | EXPECT_EQ(reference_frame->timestamp(), send_timestamp); |
| 184 | assert(reference_frame->timestamp() == send_timestamp); |
| 185 | |
| 186 | AddFrameComparison(reference_frame, &video_frame, false, render_time_ms); |
| 187 | frame_pool_.push_back(reference_frame); |
| 188 | |
pbos@webrtc.org | 9401524 | 2013-10-16 11:05:37 +0000 | [diff] [blame] | 189 | last_rendered_frame_.CopyFrame(video_frame); |
| 190 | } |
| 191 | |
pbos@webrtc.org | 994d0b7 | 2014-06-27 08:47:52 +0000 | [diff] [blame] | 192 | void Wait() { |
| 193 | EXPECT_EQ(kEventSignaled, done_->Wait(FullStackTest::kLongTimeoutMs)); |
| 194 | } |
pbos@webrtc.org | 9401524 | 2013-10-16 11:05:37 +0000 | [diff] [blame] | 195 | |
| 196 | VideoSendStreamInput* input_; |
| 197 | Transport* transport_; |
| 198 | PacketReceiver* receiver_; |
| 199 | |
| 200 | private: |
| 201 | struct FrameComparison { |
| 202 | FrameComparison(const I420VideoFrame* reference, |
| 203 | const I420VideoFrame* render, |
| 204 | bool dropped, |
| 205 | int64_t send_time_ms, |
| 206 | int64_t recv_time_ms, |
| 207 | int64_t render_time_ms) |
| 208 | : dropped(dropped), |
| 209 | send_time_ms(send_time_ms), |
| 210 | recv_time_ms(recv_time_ms), |
| 211 | render_time_ms(render_time_ms) { |
| 212 | this->reference.CopyFrame(*reference); |
| 213 | this->render.CopyFrame(*render); |
| 214 | } |
| 215 | |
| 216 | FrameComparison(const FrameComparison& compare) |
| 217 | : dropped(compare.dropped), |
| 218 | send_time_ms(compare.send_time_ms), |
| 219 | recv_time_ms(compare.recv_time_ms), |
| 220 | render_time_ms(compare.render_time_ms) { |
| 221 | this->reference.CopyFrame(compare.reference); |
| 222 | this->render.CopyFrame(compare.render); |
| 223 | } |
| 224 | |
| 225 | ~FrameComparison() {} |
| 226 | |
| 227 | I420VideoFrame reference; |
| 228 | I420VideoFrame render; |
| 229 | bool dropped; |
| 230 | int64_t send_time_ms; |
| 231 | int64_t recv_time_ms; |
| 232 | int64_t render_time_ms; |
| 233 | }; |
| 234 | |
| 235 | void AddFrameComparison(const I420VideoFrame* reference, |
| 236 | const I420VideoFrame* render, |
| 237 | bool dropped, |
pbos@webrtc.org | de1429e | 2014-04-28 13:00:21 +0000 | [diff] [blame] | 238 | int64_t render_time_ms) |
| 239 | EXCLUSIVE_LOCKS_REQUIRED(crit_) { |
pbos@webrtc.org | 9401524 | 2013-10-16 11:05:37 +0000 | [diff] [blame] | 240 | int64_t send_time_ms = send_times_[reference->timestamp()]; |
| 241 | send_times_.erase(reference->timestamp()); |
| 242 | int64_t recv_time_ms = recv_times_[reference->timestamp()]; |
| 243 | recv_times_.erase(reference->timestamp()); |
| 244 | |
| 245 | CriticalSectionScoped crit(comparison_lock_.get()); |
| 246 | comparisons_.push_back(FrameComparison(reference, |
| 247 | render, |
| 248 | dropped, |
| 249 | send_time_ms, |
| 250 | recv_time_ms, |
| 251 | render_time_ms)); |
| 252 | } |
| 253 | |
| 254 | static bool FrameComparisonThread(void* obj) { |
| 255 | return static_cast<VideoAnalyzer*>(obj)->CompareFrames(); |
| 256 | } |
| 257 | |
| 258 | bool CompareFrames() { |
| 259 | assert(frames_left_ > 0); |
| 260 | |
| 261 | I420VideoFrame reference; |
| 262 | I420VideoFrame render; |
| 263 | bool dropped; |
| 264 | int64_t send_time_ms; |
| 265 | int64_t recv_time_ms; |
| 266 | int64_t render_time_ms; |
| 267 | |
| 268 | SleepMs(10); |
| 269 | |
| 270 | while (true) { |
| 271 | { |
| 272 | CriticalSectionScoped crit(comparison_lock_.get()); |
| 273 | if (comparisons_.empty()) |
| 274 | return true; |
| 275 | reference.SwapFrame(&comparisons_.front().reference); |
| 276 | render.SwapFrame(&comparisons_.front().render); |
| 277 | dropped = comparisons_.front().dropped; |
| 278 | send_time_ms = comparisons_.front().send_time_ms; |
| 279 | recv_time_ms = comparisons_.front().recv_time_ms; |
| 280 | render_time_ms = comparisons_.front().render_time_ms; |
| 281 | comparisons_.pop_front(); |
| 282 | } |
| 283 | |
| 284 | PerformFrameComparison(&reference, |
| 285 | &render, |
| 286 | dropped, |
| 287 | send_time_ms, |
| 288 | recv_time_ms, |
| 289 | render_time_ms); |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 290 | |
| 291 | if (--frames_left_ == 0) { |
| 292 | PrintResult("psnr", psnr_, " dB"); |
| 293 | PrintResult("ssim", ssim_, ""); |
| 294 | PrintResult("sender_time", sender_time_, " ms"); |
pbos@webrtc.org | 9401524 | 2013-10-16 11:05:37 +0000 | [diff] [blame] | 295 | printf( |
| 296 | "RESULT dropped_frames: %s = %d\n", test_label_, dropped_frames_); |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 297 | PrintResult("receiver_time", receiver_time_, " ms"); |
| 298 | PrintResult("total_delay_incl_network", end_to_end_, " ms"); |
| 299 | PrintResult("time_between_rendered_frames", rendered_delta_, " ms"); |
| 300 | EXPECT_GT(psnr_.Mean(), avg_psnr_threshold_); |
| 301 | EXPECT_GT(ssim_.Mean(), avg_ssim_threshold_); |
pbos@webrtc.org | de1429e | 2014-04-28 13:00:21 +0000 | [diff] [blame] | 302 | done_->Set(); |
pbos@webrtc.org | 9401524 | 2013-10-16 11:05:37 +0000 | [diff] [blame] | 303 | |
| 304 | return false; |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 305 | } |
| 306 | } |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 307 | } |
| 308 | |
pbos@webrtc.org | 9401524 | 2013-10-16 11:05:37 +0000 | [diff] [blame] | 309 | void PerformFrameComparison(const I420VideoFrame* reference, |
| 310 | const I420VideoFrame* render, |
| 311 | bool dropped, |
| 312 | int64_t send_time_ms, |
| 313 | int64_t recv_time_ms, |
| 314 | int64_t render_time_ms) { |
| 315 | psnr_.AddSample(I420PSNR(reference, render)); |
| 316 | ssim_.AddSample(I420SSIM(reference, render)); |
| 317 | if (dropped) { |
| 318 | ++dropped_frames_; |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 319 | return; |
pbos@webrtc.org | 9401524 | 2013-10-16 11:05:37 +0000 | [diff] [blame] | 320 | } |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 321 | if (last_render_time_ != 0) |
pbos@webrtc.org | 9401524 | 2013-10-16 11:05:37 +0000 | [diff] [blame] | 322 | rendered_delta_.AddSample(render_time_ms - last_render_time_); |
| 323 | last_render_time_ = render_time_ms; |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 324 | |
pbos@webrtc.org | 9401524 | 2013-10-16 11:05:37 +0000 | [diff] [blame] | 325 | int64_t input_time_ms = reference->render_time_ms(); |
| 326 | sender_time_.AddSample(send_time_ms - input_time_ms); |
| 327 | receiver_time_.AddSample(render_time_ms - recv_time_ms); |
| 328 | end_to_end_.AddSample(render_time_ms - input_time_ms); |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 329 | } |
| 330 | |
| 331 | void PrintResult(const char* result_type, |
| 332 | test::Statistics stats, |
| 333 | const char* unit) { |
| 334 | printf("RESULT %s: %s = {%f, %f}%s\n", |
| 335 | result_type, |
| 336 | test_label_, |
| 337 | stats.Mean(), |
| 338 | stats.StandardDeviation(), |
| 339 | unit); |
| 340 | } |
| 341 | |
pbos@webrtc.org | de1429e | 2014-04-28 13:00:21 +0000 | [diff] [blame] | 342 | const char* const test_label_; |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 343 | test::Statistics sender_time_; |
| 344 | test::Statistics receiver_time_; |
| 345 | test::Statistics psnr_; |
| 346 | test::Statistics ssim_; |
| 347 | test::Statistics end_to_end_; |
| 348 | test::Statistics rendered_delta_; |
pbos@webrtc.org | 9401524 | 2013-10-16 11:05:37 +0000 | [diff] [blame] | 349 | int frames_left_; |
pbos@webrtc.org | de1429e | 2014-04-28 13:00:21 +0000 | [diff] [blame] | 350 | int dropped_frames_; |
| 351 | int64_t last_render_time_; |
| 352 | uint32_t rtp_timestamp_delta_; |
| 353 | |
| 354 | const scoped_ptr<CriticalSectionWrapper> crit_; |
| 355 | std::deque<I420VideoFrame*> frames_ GUARDED_BY(crit_); |
| 356 | std::deque<I420VideoFrame*> frame_pool_ GUARDED_BY(crit_); |
| 357 | I420VideoFrame last_rendered_frame_ GUARDED_BY(crit_); |
| 358 | std::map<uint32_t, int64_t> send_times_ GUARDED_BY(crit_); |
| 359 | std::map<uint32_t, int64_t> recv_times_ GUARDED_BY(crit_); |
| 360 | I420VideoFrame* first_send_frame_ GUARDED_BY(crit_); |
| 361 | double avg_psnr_threshold_ GUARDED_BY(crit_); |
| 362 | double avg_ssim_threshold_ GUARDED_BY(crit_); |
| 363 | |
| 364 | const scoped_ptr<CriticalSectionWrapper> comparison_lock_; |
| 365 | const scoped_ptr<ThreadWrapper> comparison_thread_; |
| 366 | std::deque<FrameComparison> comparisons_ GUARDED_BY(comparison_lock_); |
| 367 | const scoped_ptr<EventWrapper> done_; |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 368 | }; |
| 369 | |
pbos@webrtc.org | 994d0b7 | 2014-06-27 08:47:52 +0000 | [diff] [blame] | 370 | void FullStackTest::TestWithoutPacketLoss(const FullStackTestParams& params) { |
pbos@webrtc.org | 9668467 | 2013-08-12 12:59:04 +0000 | [diff] [blame] | 371 | test::DirectTransport transport; |
pbos@webrtc.org | 9401524 | 2013-10-16 11:05:37 +0000 | [diff] [blame] | 372 | VideoAnalyzer analyzer(NULL, |
| 373 | &transport, |
| 374 | params.test_label, |
| 375 | params.avg_psnr_threshold, |
| 376 | params.avg_ssim_threshold, |
pbos@webrtc.org | 023b101 | 2014-05-13 11:26:40 +0000 | [diff] [blame] | 377 | kFullStackTestDurationSecs * params.clip.fps); |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 378 | |
pbos@webrtc.org | 994d0b7 | 2014-06-27 08:47:52 +0000 | [diff] [blame] | 379 | CreateCalls(Call::Config(&analyzer), Call::Config(&analyzer)); |
mflodman@webrtc.org | 6879c8a | 2013-07-23 11:35:00 +0000 | [diff] [blame] | 380 | |
pbos@webrtc.org | 994d0b7 | 2014-06-27 08:47:52 +0000 | [diff] [blame] | 381 | analyzer.SetReceiver(receiver_call_->Receiver()); |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 382 | transport.SetReceiver(&analyzer); |
| 383 | |
pbos@webrtc.org | 994d0b7 | 2014-06-27 08:47:52 +0000 | [diff] [blame] | 384 | CreateSendConfig(1); |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 385 | |
pbos@webrtc.org | f577ae9 | 2014-03-19 08:43:57 +0000 | [diff] [blame] | 386 | scoped_ptr<VP8Encoder> encoder(VP8Encoder::Create()); |
pbos@webrtc.org | 994d0b7 | 2014-06-27 08:47:52 +0000 | [diff] [blame] | 387 | send_config_.encoder_settings.encoder = encoder.get(); |
| 388 | send_config_.encoder_settings.payload_name = "VP8"; |
| 389 | send_config_.encoder_settings.payload_type = 124; |
| 390 | |
| 391 | VideoStream* stream = &video_streams_[0]; |
pbos@webrtc.org | f577ae9 | 2014-03-19 08:43:57 +0000 | [diff] [blame] | 392 | stream->width = params.clip.width; |
| 393 | stream->height = params.clip.height; |
| 394 | stream->min_bitrate_bps = stream->target_bitrate_bps = |
| 395 | stream->max_bitrate_bps = params.bitrate * 1000; |
| 396 | stream->max_framerate = params.clip.fps; |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 397 | |
pbos@webrtc.org | 994d0b7 | 2014-06-27 08:47:52 +0000 | [diff] [blame] | 398 | CreateMatchingReceiveConfigs(); |
pbos@webrtc.org | be9d2a4 | 2014-06-30 13:19:09 +0000 | [diff] [blame] | 399 | receive_configs_[0].renderer = &analyzer; |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 400 | |
pbos@webrtc.org | 994d0b7 | 2014-06-27 08:47:52 +0000 | [diff] [blame] | 401 | CreateStreams(); |
| 402 | analyzer.input_ = send_stream_->Input(); |
| 403 | |
| 404 | frame_generator_capturer_.reset( |
andresp@webrtc.org | ab65495 | 2013-09-19 12:14:03 +0000 | [diff] [blame] | 405 | test::FrameGeneratorCapturer::CreateFromYuvFile( |
pbos@webrtc.org | 4c96601 | 2013-08-21 12:07:37 +0000 | [diff] [blame] | 406 | &analyzer, |
andresp@webrtc.org | ab65495 | 2013-09-19 12:14:03 +0000 | [diff] [blame] | 407 | test::ResourcePath(params.clip.name, "yuv").c_str(), |
| 408 | params.clip.width, |
| 409 | params.clip.height, |
| 410 | params.clip.fps, |
| 411 | Clock::GetRealTimeClock())); |
pbos@webrtc.org | 994d0b7 | 2014-06-27 08:47:52 +0000 | [diff] [blame] | 412 | |
| 413 | ASSERT_TRUE(frame_generator_capturer_.get() != NULL) |
pbos@webrtc.org | 9401524 | 2013-10-16 11:05:37 +0000 | [diff] [blame] | 414 | << "Could not create capturer for " << params.clip.name |
| 415 | << ".yuv. Is this resource file present?"; |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 416 | |
pbos@webrtc.org | 994d0b7 | 2014-06-27 08:47:52 +0000 | [diff] [blame] | 417 | Start(); |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 418 | |
| 419 | analyzer.Wait(); |
| 420 | |
pbos@webrtc.org | 9668467 | 2013-08-12 12:59:04 +0000 | [diff] [blame] | 421 | transport.StopSending(); |
pbos@webrtc.org | 994d0b7 | 2014-06-27 08:47:52 +0000 | [diff] [blame] | 422 | |
| 423 | Stop(); |
| 424 | |
| 425 | DestroyStreams(); |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 426 | } |
| 427 | |
pbos@webrtc.org | 994d0b7 | 2014-06-27 08:47:52 +0000 | [diff] [blame] | 428 | FullStackTestParams paris_qcif = {"net_delay_0_0_plr_0", |
| 429 | {"paris_qcif", 176, 144, 30}, |
| 430 | 300, |
| 431 | 36.0, |
| 432 | 0.96}; |
| 433 | |
| 434 | // TODO(pbos): Decide on psnr/ssim thresholds for foreman_cif. |
| 435 | FullStackTestParams foreman_cif = {"foreman_cif_net_delay_0_0_plr_0", |
| 436 | {"foreman_cif", 352, 288, 30}, |
| 437 | 700, |
| 438 | 0.0, |
| 439 | 0.0}; |
| 440 | |
| 441 | TEST_F(FullStackTest, ParisQcifWithoutPacketLoss) { |
| 442 | TestWithoutPacketLoss(paris_qcif); |
| 443 | } |
| 444 | |
| 445 | TEST_F(FullStackTest, ForemanCifWithoutPacketLoss) { |
| 446 | TestWithoutPacketLoss(foreman_cif); |
| 447 | } |
pbos@webrtc.org | af8d5af | 2013-07-09 08:02:33 +0000 | [diff] [blame] | 448 | |
| 449 | } // namespace webrtc |