blob: b733a1d2cf0ba01811094f2ea03528c0651b7b37 [file] [log] [blame]
stefan@webrtc.org5f284982012-06-28 07:51:16 +00001/*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Jonas Olssona4d87372019-07-05 19:08:33 +020011#include "video/stream_synchronization.h"
12
kjellander@webrtc.org0fcaf992015-11-26 15:24:52 +010013#include <algorithm>
14
Åsa Persson8fe22fa2019-11-18 14:10:56 +010015#include "system_wrappers/include/clock.h"
Yves Gerey3e707812018-11-28 16:47:49 +010016#include "system_wrappers/include/ntp_time.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020017#include "test/gtest.h"
stefan@webrtc.org5f284982012-06-28 07:51:16 +000018
19namespace webrtc {
Åsa Persson8fe22fa2019-11-18 14:10:56 +010020namespace {
Åsa Persson0e578582020-02-03 11:13:20 +010021constexpr int kMaxChangeMs = 80; // From stream_synchronization.cc
Åsa Persson8fe22fa2019-11-18 14:10:56 +010022constexpr int kDefaultAudioFrequency = 8000;
23constexpr int kDefaultVideoFrequency = 90000;
24constexpr int kSmoothingFilter = 4 * 2;
25} // namespace
stefan@webrtc.org5f284982012-06-28 07:51:16 +000026
27class StreamSynchronizationTest : public ::testing::Test {
Åsa Persson8fe22fa2019-11-18 14:10:56 +010028 public:
29 StreamSynchronizationTest()
30 : sync_(0, 0), clock_sender_(98765000), clock_receiver_(43210000) {}
31
stefan@webrtc.org5f284982012-06-28 07:51:16 +000032 protected:
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +000033 // Generates the necessary RTCP measurements and RTP timestamps and computes
34 // the audio and video delays needed to get the two streams in sync.
Artem Titovab30d722021-07-27 16:22:11 +020035 // `audio_delay_ms` and `video_delay_ms` are the number of milliseconds after
Åsa Persson0e578582020-02-03 11:13:20 +010036 // capture which the frames are received.
Artem Titovab30d722021-07-27 16:22:11 +020037 // `current_audio_delay_ms` is the number of milliseconds which audio is
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +000038 // currently being delayed by the receiver.
39 bool DelayedStreams(int audio_delay_ms,
40 int video_delay_ms,
41 int current_audio_delay_ms,
Åsa Persson0e578582020-02-03 11:13:20 +010042 int* total_audio_delay_ms,
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +000043 int* total_video_delay_ms) {
Yves Gerey665174f2018-06-19 15:03:05 +020044 int audio_frequency =
45 static_cast<int>(kDefaultAudioFrequency * audio_clock_drift_ + 0.5);
Yves Gerey665174f2018-06-19 15:03:05 +020046 int video_frequency =
47 static_cast<int>(kDefaultVideoFrequency * video_clock_drift_ + 0.5);
Åsa Persson8fe22fa2019-11-18 14:10:56 +010048
49 // Generate NTP/RTP timestamp pair for both streams corresponding to RTCP.
stefan@webrtc.org5f284982012-06-28 07:51:16 +000050 StreamSynchronization::Measurements audio;
51 StreamSynchronization::Measurements video;
Åsa Persson8fe22fa2019-11-18 14:10:56 +010052 NtpTime ntp_time = clock_sender_.CurrentNtpTime();
asaperssonfe50b4d2016-12-22 07:53:51 -080053 uint32_t rtp_timestamp =
Åsa Persson8fe22fa2019-11-18 14:10:56 +010054 clock_sender_.CurrentTime().ms() * audio_frequency / 1000;
Danil Chapovalovae4fb612022-03-14 12:31:46 +010055 EXPECT_EQ(audio.rtp_to_ntp.UpdateMeasurements(ntp_time, rtp_timestamp),
56 RtpToNtpEstimator::kNewMeasurement);
Åsa Persson8fe22fa2019-11-18 14:10:56 +010057 clock_sender_.AdvanceTimeMilliseconds(100);
58 clock_receiver_.AdvanceTimeMilliseconds(100);
59 ntp_time = clock_sender_.CurrentNtpTime();
60 rtp_timestamp = clock_sender_.CurrentTime().ms() * video_frequency / 1000;
Danil Chapovalovae4fb612022-03-14 12:31:46 +010061 EXPECT_EQ(video.rtp_to_ntp.UpdateMeasurements(ntp_time, rtp_timestamp),
62 RtpToNtpEstimator::kNewMeasurement);
Åsa Persson8fe22fa2019-11-18 14:10:56 +010063 clock_sender_.AdvanceTimeMilliseconds(900);
64 clock_receiver_.AdvanceTimeMilliseconds(900);
65 ntp_time = clock_sender_.CurrentNtpTime();
66 rtp_timestamp = clock_sender_.CurrentTime().ms() * audio_frequency / 1000;
Danil Chapovalovae4fb612022-03-14 12:31:46 +010067 EXPECT_EQ(audio.rtp_to_ntp.UpdateMeasurements(ntp_time, rtp_timestamp),
68 RtpToNtpEstimator::kNewMeasurement);
Åsa Persson8fe22fa2019-11-18 14:10:56 +010069 clock_sender_.AdvanceTimeMilliseconds(100);
70 clock_receiver_.AdvanceTimeMilliseconds(100);
71 ntp_time = clock_sender_.CurrentNtpTime();
72 rtp_timestamp = clock_sender_.CurrentTime().ms() * video_frequency / 1000;
Danil Chapovalovae4fb612022-03-14 12:31:46 +010073 EXPECT_EQ(video.rtp_to_ntp.UpdateMeasurements(ntp_time, rtp_timestamp),
74 RtpToNtpEstimator::kNewMeasurement);
Åsa Persson8fe22fa2019-11-18 14:10:56 +010075 clock_sender_.AdvanceTimeMilliseconds(900);
76 clock_receiver_.AdvanceTimeMilliseconds(900);
stefan@webrtc.org5f284982012-06-28 07:51:16 +000077
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +000078 // Capture an audio and a video frame at the same time.
asaperssonb7e7b492016-11-17 02:27:14 -080079 audio.latest_timestamp =
Åsa Persson8fe22fa2019-11-18 14:10:56 +010080 clock_sender_.CurrentTime().ms() * audio_frequency / 1000;
asaperssonb7e7b492016-11-17 02:27:14 -080081 video.latest_timestamp =
Åsa Persson8fe22fa2019-11-18 14:10:56 +010082 clock_sender_.CurrentTime().ms() * video_frequency / 1000;
stefan@webrtc.org5f284982012-06-28 07:51:16 +000083
84 if (audio_delay_ms > video_delay_ms) {
85 // Audio later than video.
Åsa Persson8fe22fa2019-11-18 14:10:56 +010086 clock_receiver_.AdvanceTimeMilliseconds(video_delay_ms);
87 video.latest_receive_time_ms = clock_receiver_.CurrentTime().ms();
88 clock_receiver_.AdvanceTimeMilliseconds(audio_delay_ms - video_delay_ms);
89 audio.latest_receive_time_ms = clock_receiver_.CurrentTime().ms();
stefan@webrtc.org5f284982012-06-28 07:51:16 +000090 } else {
91 // Video later than audio.
Åsa Persson8fe22fa2019-11-18 14:10:56 +010092 clock_receiver_.AdvanceTimeMilliseconds(audio_delay_ms);
93 audio.latest_receive_time_ms = clock_receiver_.CurrentTime().ms();
94 clock_receiver_.AdvanceTimeMilliseconds(video_delay_ms - audio_delay_ms);
95 video.latest_receive_time_ms = clock_receiver_.CurrentTime().ms();
stefan@webrtc.org5f284982012-06-28 07:51:16 +000096 }
Åsa Persson0e578582020-02-03 11:13:20 +010097
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +000098 int relative_delay_ms;
Åsa Persson0e578582020-02-03 11:13:20 +010099 EXPECT_TRUE(StreamSynchronization::ComputeRelativeDelay(
100 audio, video, &relative_delay_ms));
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000101 EXPECT_EQ(video_delay_ms - audio_delay_ms, relative_delay_ms);
Åsa Persson0e578582020-02-03 11:13:20 +0100102
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100103 return sync_.ComputeDelays(relative_delay_ms, current_audio_delay_ms,
Åsa Persson0e578582020-02-03 11:13:20 +0100104 total_audio_delay_ms, total_video_delay_ms);
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000105 }
106
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000107 // Simulate audio playback 300 ms after capture and video rendering 100 ms
108 // after capture. Verify that the correct extra delays are calculated for
109 // audio and video, and that they change correctly when we simulate that
110 // NetEQ or the VCM adds more delay to the streams.
Åsa Persson0e578582020-02-03 11:13:20 +0100111 void BothDelayedAudioLaterTest(int base_target_delay_ms) {
112 const int kAudioDelayMs = base_target_delay_ms + 300;
113 const int kVideoDelayMs = base_target_delay_ms + 100;
114 int current_audio_delay_ms = base_target_delay_ms;
115 int total_audio_delay_ms = 0;
116 int total_video_delay_ms = base_target_delay_ms;
117 int filtered_move = (kAudioDelayMs - kVideoDelayMs) / kSmoothingFilter;
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000118
Åsa Persson0e578582020-02-03 11:13:20 +0100119 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
120 current_audio_delay_ms, &total_audio_delay_ms,
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000121 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 11:13:20 +0100122 EXPECT_EQ(base_target_delay_ms + filtered_move, total_video_delay_ms);
123 EXPECT_EQ(base_target_delay_ms, total_audio_delay_ms);
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000124
Åsa Persson0e578582020-02-03 11:13:20 +0100125 // Set new current delay.
126 current_audio_delay_ms = total_audio_delay_ms;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100127 clock_sender_.AdvanceTimeMilliseconds(1000);
128 clock_receiver_.AdvanceTimeMilliseconds(
Åsa Persson0e578582020-02-03 11:13:20 +0100129 1000 - std::max(kAudioDelayMs, kVideoDelayMs));
130 // Simulate base_target_delay_ms minimum delay in the VCM.
131 total_video_delay_ms = base_target_delay_ms;
132 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
133 current_audio_delay_ms, &total_audio_delay_ms,
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000134 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 11:13:20 +0100135 EXPECT_EQ(base_target_delay_ms + 2 * filtered_move, total_video_delay_ms);
136 EXPECT_EQ(base_target_delay_ms, total_audio_delay_ms);
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000137
Åsa Persson0e578582020-02-03 11:13:20 +0100138 // Set new current delay.
139 current_audio_delay_ms = total_audio_delay_ms;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100140 clock_sender_.AdvanceTimeMilliseconds(1000);
141 clock_receiver_.AdvanceTimeMilliseconds(
Åsa Persson0e578582020-02-03 11:13:20 +0100142 1000 - std::max(kAudioDelayMs, kVideoDelayMs));
143 // Simulate base_target_delay_ms minimum delay in the VCM.
144 total_video_delay_ms = base_target_delay_ms;
145 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
146 current_audio_delay_ms, &total_audio_delay_ms,
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000147 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 11:13:20 +0100148 EXPECT_EQ(base_target_delay_ms + 3 * filtered_move, total_video_delay_ms);
149 EXPECT_EQ(base_target_delay_ms, total_audio_delay_ms);
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000150
151 // Simulate that NetEQ introduces some audio delay.
Åsa Persson0e578582020-02-03 11:13:20 +0100152 const int kNeteqDelayIncrease = 50;
153 current_audio_delay_ms = base_target_delay_ms + kNeteqDelayIncrease;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100154 clock_sender_.AdvanceTimeMilliseconds(1000);
155 clock_receiver_.AdvanceTimeMilliseconds(
Åsa Persson0e578582020-02-03 11:13:20 +0100156 1000 - std::max(kAudioDelayMs, kVideoDelayMs));
157 // Simulate base_target_delay_ms minimum delay in the VCM.
158 total_video_delay_ms = base_target_delay_ms;
159 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
160 current_audio_delay_ms, &total_audio_delay_ms,
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000161 &total_video_delay_ms));
pwestin@webrtc.org63117332013-04-22 18:57:14 +0000162 filtered_move = 3 * filtered_move +
Åsa Persson0e578582020-02-03 11:13:20 +0100163 (kNeteqDelayIncrease + kAudioDelayMs - kVideoDelayMs) /
Yves Gerey665174f2018-06-19 15:03:05 +0200164 kSmoothingFilter;
Åsa Persson0e578582020-02-03 11:13:20 +0100165 EXPECT_EQ(base_target_delay_ms + filtered_move, total_video_delay_ms);
166 EXPECT_EQ(base_target_delay_ms, total_audio_delay_ms);
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000167
168 // Simulate that NetEQ reduces its delay.
Åsa Persson0e578582020-02-03 11:13:20 +0100169 const int kNeteqDelayDecrease = 10;
170 current_audio_delay_ms = base_target_delay_ms + kNeteqDelayDecrease;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100171 clock_sender_.AdvanceTimeMilliseconds(1000);
172 clock_receiver_.AdvanceTimeMilliseconds(
Åsa Persson0e578582020-02-03 11:13:20 +0100173 1000 - std::max(kAudioDelayMs, kVideoDelayMs));
174 // Simulate base_target_delay_ms minimum delay in the VCM.
175 total_video_delay_ms = base_target_delay_ms;
176 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
177 current_audio_delay_ms, &total_audio_delay_ms,
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000178 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 11:13:20 +0100179 filtered_move =
180 filtered_move + (kNeteqDelayDecrease + kAudioDelayMs - kVideoDelayMs) /
181 kSmoothingFilter;
182 EXPECT_EQ(base_target_delay_ms + filtered_move, total_video_delay_ms);
183 EXPECT_EQ(base_target_delay_ms, total_audio_delay_ms);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000184 }
185
Åsa Persson0e578582020-02-03 11:13:20 +0100186 void BothDelayedVideoLaterTest(int base_target_delay_ms) {
187 const int kAudioDelayMs = base_target_delay_ms + 100;
188 const int kVideoDelayMs = base_target_delay_ms + 300;
189 int current_audio_delay_ms = base_target_delay_ms;
190 int total_audio_delay_ms = 0;
191 int total_video_delay_ms = base_target_delay_ms;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000192
Åsa Persson0e578582020-02-03 11:13:20 +0100193 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
194 current_audio_delay_ms, &total_audio_delay_ms,
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000195 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 11:13:20 +0100196 EXPECT_EQ(base_target_delay_ms, total_video_delay_ms);
197 // The audio delay is not allowed to change more than this.
198 EXPECT_GE(base_target_delay_ms + kMaxChangeMs, total_audio_delay_ms);
199 int last_total_audio_delay_ms = total_audio_delay_ms;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000200
Åsa Persson0e578582020-02-03 11:13:20 +0100201 // Set new current audio delay.
202 current_audio_delay_ms = total_audio_delay_ms;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100203 clock_sender_.AdvanceTimeMilliseconds(1000);
204 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 11:13:20 +0100205 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
206 current_audio_delay_ms, &total_audio_delay_ms,
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000207 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 11:13:20 +0100208 EXPECT_EQ(base_target_delay_ms, total_video_delay_ms);
209 EXPECT_EQ(last_total_audio_delay_ms +
210 MaxAudioDelayChangeMs(
Yves Gerey665174f2018-06-19 15:03:05 +0200211 current_audio_delay_ms,
Åsa Persson0e578582020-02-03 11:13:20 +0100212 base_target_delay_ms + kVideoDelayMs - kAudioDelayMs),
213 total_audio_delay_ms);
214 last_total_audio_delay_ms = total_audio_delay_ms;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000215
Åsa Persson0e578582020-02-03 11:13:20 +0100216 // Set new current audio delay.
217 current_audio_delay_ms = total_audio_delay_ms;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100218 clock_sender_.AdvanceTimeMilliseconds(1000);
219 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 11:13:20 +0100220 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
221 current_audio_delay_ms, &total_audio_delay_ms,
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000222 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 11:13:20 +0100223 EXPECT_EQ(base_target_delay_ms, total_video_delay_ms);
224 EXPECT_EQ(last_total_audio_delay_ms +
225 MaxAudioDelayChangeMs(
Yves Gerey665174f2018-06-19 15:03:05 +0200226 current_audio_delay_ms,
Åsa Persson0e578582020-02-03 11:13:20 +0100227 base_target_delay_ms + kVideoDelayMs - kAudioDelayMs),
228 total_audio_delay_ms);
229 last_total_audio_delay_ms = total_audio_delay_ms;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000230
231 // Simulate that NetEQ for some reason reduced the delay.
Åsa Persson0e578582020-02-03 11:13:20 +0100232 current_audio_delay_ms = base_target_delay_ms + 10;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100233 clock_sender_.AdvanceTimeMilliseconds(1000);
234 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 11:13:20 +0100235 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
236 current_audio_delay_ms, &total_audio_delay_ms,
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000237 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 11:13:20 +0100238 EXPECT_EQ(base_target_delay_ms, total_video_delay_ms);
239 EXPECT_EQ(last_total_audio_delay_ms +
240 MaxAudioDelayChangeMs(
Yves Gerey665174f2018-06-19 15:03:05 +0200241 current_audio_delay_ms,
Åsa Persson0e578582020-02-03 11:13:20 +0100242 base_target_delay_ms + kVideoDelayMs - kAudioDelayMs),
243 total_audio_delay_ms);
244 last_total_audio_delay_ms = total_audio_delay_ms;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000245
246 // Simulate that NetEQ for some reason significantly increased the delay.
Åsa Persson0e578582020-02-03 11:13:20 +0100247 current_audio_delay_ms = base_target_delay_ms + 350;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100248 clock_sender_.AdvanceTimeMilliseconds(1000);
249 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 11:13:20 +0100250 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
251 current_audio_delay_ms, &total_audio_delay_ms,
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000252 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 11:13:20 +0100253 EXPECT_EQ(base_target_delay_ms, total_video_delay_ms);
254 EXPECT_EQ(last_total_audio_delay_ms +
255 MaxAudioDelayChangeMs(
Yves Gerey665174f2018-06-19 15:03:05 +0200256 current_audio_delay_ms,
Åsa Persson0e578582020-02-03 11:13:20 +0100257 base_target_delay_ms + kVideoDelayMs - kAudioDelayMs),
258 total_audio_delay_ms);
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000259 }
260
Åsa Persson0e578582020-02-03 11:13:20 +0100261 int MaxAudioDelayChangeMs(int current_audio_delay_ms, int delay_ms) const {
262 int diff_ms = (delay_ms - current_audio_delay_ms) / kSmoothingFilter;
263 diff_ms = std::min(diff_ms, kMaxChangeMs);
264 diff_ms = std::max(diff_ms, -kMaxChangeMs);
265 return diff_ms;
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000266 }
267
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100268 StreamSynchronization sync_;
269 SimulatedClock clock_sender_;
270 SimulatedClock clock_receiver_;
271 double audio_clock_drift_ = 1.0;
272 double video_clock_drift_ = 1.0;
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000273};
274
275TEST_F(StreamSynchronizationTest, NoDelay) {
Åsa Persson0e578582020-02-03 11:13:20 +0100276 int total_audio_delay_ms = 0;
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000277 int total_video_delay_ms = 0;
278
Åsa Persson0e578582020-02-03 11:13:20 +0100279 EXPECT_FALSE(DelayedStreams(/*audio_delay_ms=*/0, /*video_delay_ms=*/0,
280 /*current_audio_delay_ms=*/0,
281 &total_audio_delay_ms, &total_video_delay_ms));
282 EXPECT_EQ(0, total_audio_delay_ms);
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000283 EXPECT_EQ(0, total_video_delay_ms);
284}
285
Åsa Persson0e578582020-02-03 11:13:20 +0100286TEST_F(StreamSynchronizationTest, VideoDelayed) {
287 const int kAudioDelayMs = 200;
288 int total_audio_delay_ms = 0;
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000289 int total_video_delay_ms = 0;
290
Åsa Persson0e578582020-02-03 11:13:20 +0100291 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, /*video_delay_ms=*/0,
292 /*current_audio_delay_ms=*/0,
293 &total_audio_delay_ms, &total_video_delay_ms));
294 EXPECT_EQ(0, total_audio_delay_ms);
295 // The delay is not allowed to change more than this.
296 EXPECT_EQ(kAudioDelayMs / kSmoothingFilter, total_video_delay_ms);
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000297
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000298 // Simulate 0 minimum delay in the VCM.
299 total_video_delay_ms = 0;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100300 clock_sender_.AdvanceTimeMilliseconds(1000);
301 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 11:13:20 +0100302 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, /*video_delay_ms=*/0,
303 /*current_audio_delay_ms=*/0,
304 &total_audio_delay_ms, &total_video_delay_ms));
305 EXPECT_EQ(0, total_audio_delay_ms);
306 EXPECT_EQ(2 * kAudioDelayMs / kSmoothingFilter, total_video_delay_ms);
307
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000308 // Simulate 0 minimum delay in the VCM.
309 total_video_delay_ms = 0;
Åsa Persson0e578582020-02-03 11:13:20 +0100310 clock_sender_.AdvanceTimeMilliseconds(1000);
311 clock_receiver_.AdvanceTimeMilliseconds(800);
312 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, /*video_delay_ms=*/0,
313 /*current_audio_delay_ms=*/0,
314 &total_audio_delay_ms, &total_video_delay_ms));
315 EXPECT_EQ(0, total_audio_delay_ms);
316 EXPECT_EQ(3 * kAudioDelayMs / kSmoothingFilter, total_video_delay_ms);
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000317}
318
Åsa Persson0e578582020-02-03 11:13:20 +0100319TEST_F(StreamSynchronizationTest, AudioDelayed) {
320 const int kVideoDelayMs = 200;
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000321 int current_audio_delay_ms = 0;
Åsa Persson0e578582020-02-03 11:13:20 +0100322 int total_audio_delay_ms = 0;
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000323 int total_video_delay_ms = 0;
324
Åsa Persson0e578582020-02-03 11:13:20 +0100325 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
326 current_audio_delay_ms, &total_audio_delay_ms,
327 &total_video_delay_ms));
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000328 EXPECT_EQ(0, total_video_delay_ms);
Åsa Persson0e578582020-02-03 11:13:20 +0100329 // The delay is not allowed to change more than this.
330 EXPECT_EQ(kVideoDelayMs / kSmoothingFilter, total_audio_delay_ms);
331 int last_total_audio_delay_ms = total_audio_delay_ms;
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000332
Åsa Persson0e578582020-02-03 11:13:20 +0100333 // Set new current audio delay.
334 current_audio_delay_ms = total_audio_delay_ms;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100335 clock_sender_.AdvanceTimeMilliseconds(1000);
336 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 11:13:20 +0100337 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
338 current_audio_delay_ms, &total_audio_delay_ms,
339 &total_video_delay_ms));
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000340 EXPECT_EQ(0, total_video_delay_ms);
Åsa Persson0e578582020-02-03 11:13:20 +0100341 EXPECT_EQ(last_total_audio_delay_ms +
342 MaxAudioDelayChangeMs(current_audio_delay_ms, kVideoDelayMs),
343 total_audio_delay_ms);
344 last_total_audio_delay_ms = total_audio_delay_ms;
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000345
Åsa Persson0e578582020-02-03 11:13:20 +0100346 // Set new current audio delay.
347 current_audio_delay_ms = total_audio_delay_ms;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100348 clock_sender_.AdvanceTimeMilliseconds(1000);
349 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 11:13:20 +0100350 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
351 current_audio_delay_ms, &total_audio_delay_ms,
352 &total_video_delay_ms));
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000353 EXPECT_EQ(0, total_video_delay_ms);
Åsa Persson0e578582020-02-03 11:13:20 +0100354 EXPECT_EQ(last_total_audio_delay_ms +
355 MaxAudioDelayChangeMs(current_audio_delay_ms, kVideoDelayMs),
356 total_audio_delay_ms);
357 last_total_audio_delay_ms = total_audio_delay_ms;
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000358
359 // Simulate that NetEQ for some reason reduced the delay.
pwestin@webrtc.org63117332013-04-22 18:57:14 +0000360 current_audio_delay_ms = 10;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100361 clock_sender_.AdvanceTimeMilliseconds(1000);
362 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 11:13:20 +0100363 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
364 current_audio_delay_ms, &total_audio_delay_ms,
365 &total_video_delay_ms));
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000366 EXPECT_EQ(0, total_video_delay_ms);
Åsa Persson0e578582020-02-03 11:13:20 +0100367 EXPECT_EQ(last_total_audio_delay_ms +
368 MaxAudioDelayChangeMs(current_audio_delay_ms, kVideoDelayMs),
369 total_audio_delay_ms);
370 last_total_audio_delay_ms = total_audio_delay_ms;
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000371
372 // Simulate that NetEQ for some reason significantly increased the delay.
pwestin@webrtc.org63117332013-04-22 18:57:14 +0000373 current_audio_delay_ms = 350;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100374 clock_sender_.AdvanceTimeMilliseconds(1000);
375 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 11:13:20 +0100376 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
377 current_audio_delay_ms, &total_audio_delay_ms,
378 &total_video_delay_ms));
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000379 EXPECT_EQ(0, total_video_delay_ms);
Åsa Persson0e578582020-02-03 11:13:20 +0100380 EXPECT_EQ(last_total_audio_delay_ms +
381 MaxAudioDelayChangeMs(current_audio_delay_ms, kVideoDelayMs),
382 total_audio_delay_ms);
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000383}
384
Shyam Sadhwani986e7452020-09-14 10:12:54 -0700385TEST_F(StreamSynchronizationTest, NoAudioIncomingUnboundedIncrease) {
386 // Test how audio delay can grow unbounded when audio stops coming in.
387 // This is handled in caller of RtpStreamsSynchronizer, for example in
388 // RtpStreamsSynchronizer by not updating delays when audio samples stop
389 // coming in.
390 const int kVideoDelayMs = 300;
391 const int kAudioDelayMs = 100;
392 int current_audio_delay_ms = kAudioDelayMs;
393 int total_audio_delay_ms = 0;
394 int total_video_delay_ms = 0;
395
396 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
397 current_audio_delay_ms, &total_audio_delay_ms,
398 &total_video_delay_ms));
399 EXPECT_EQ(0, total_video_delay_ms);
400 // The delay is not allowed to change more than this.
401 EXPECT_EQ((kVideoDelayMs - kAudioDelayMs) / kSmoothingFilter,
402 total_audio_delay_ms);
403 int last_total_audio_delay_ms = total_audio_delay_ms;
404
405 // Set new current audio delay: simulate audio samples are flowing in.
406 current_audio_delay_ms = total_audio_delay_ms;
407
408 clock_sender_.AdvanceTimeMilliseconds(1000);
409 clock_receiver_.AdvanceTimeMilliseconds(1000);
410 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
411 current_audio_delay_ms, &total_audio_delay_ms,
412 &total_video_delay_ms));
413 EXPECT_EQ(0, total_video_delay_ms);
414 EXPECT_EQ(last_total_audio_delay_ms +
415 MaxAudioDelayChangeMs(current_audio_delay_ms, kVideoDelayMs),
416 total_audio_delay_ms);
417 last_total_audio_delay_ms = total_audio_delay_ms;
418
419 // Simulate no incoming audio by not update audio delay.
420 const int kSimulationSecs = 300; // 5min
421 const int kMaxDeltaDelayMs = 10000; // max delay for audio in webrtc
422 for (auto time_secs = 0; time_secs < kSimulationSecs; time_secs++) {
423 clock_sender_.AdvanceTimeMilliseconds(1000);
424 clock_receiver_.AdvanceTimeMilliseconds(1000);
425 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
426 current_audio_delay_ms, &total_audio_delay_ms,
427 &total_video_delay_ms));
428 EXPECT_EQ(0, total_video_delay_ms);
429
430 // Audio delay does not go above kMaxDeltaDelayMs.
431 EXPECT_EQ(std::min(kMaxDeltaDelayMs,
432 last_total_audio_delay_ms +
433 MaxAudioDelayChangeMs(current_audio_delay_ms,
434 kVideoDelayMs)),
435 total_audio_delay_ms);
436 last_total_audio_delay_ms = total_audio_delay_ms;
437 }
438 // By now the audio delay has grown unbounded to kMaxDeltaDelayMs.
439 EXPECT_EQ(kMaxDeltaDelayMs, last_total_audio_delay_ms);
440}
441
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000442TEST_F(StreamSynchronizationTest, BothDelayedVideoLater) {
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000443 BothDelayedVideoLaterTest(0);
444}
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000445
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000446TEST_F(StreamSynchronizationTest, BothDelayedVideoLaterAudioClockDrift) {
447 audio_clock_drift_ = 1.05;
448 BothDelayedVideoLaterTest(0);
449}
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000450
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000451TEST_F(StreamSynchronizationTest, BothDelayedVideoLaterVideoClockDrift) {
452 video_clock_drift_ = 1.05;
453 BothDelayedVideoLaterTest(0);
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000454}
455
456TEST_F(StreamSynchronizationTest, BothDelayedAudioLater) {
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000457 BothDelayedAudioLaterTest(0);
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000458}
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000459
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000460TEST_F(StreamSynchronizationTest, BothDelayedAudioClockDrift) {
461 audio_clock_drift_ = 1.05;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000462 BothDelayedAudioLaterTest(0);
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000463}
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000464
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000465TEST_F(StreamSynchronizationTest, BothDelayedVideoClockDrift) {
466 video_clock_drift_ = 1.05;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000467 BothDelayedAudioLaterTest(0);
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000468}
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000469
Åsa Persson0e578582020-02-03 11:13:20 +0100470TEST_F(StreamSynchronizationTest, BothEquallyDelayed) {
471 const int kDelayMs = 2000;
472 int current_audio_delay_ms = kDelayMs;
473 int total_audio_delay_ms = 0;
474 int total_video_delay_ms = kDelayMs;
475 // In sync, expect no change.
476 EXPECT_FALSE(DelayedStreams(kDelayMs, kDelayMs, current_audio_delay_ms,
477 &total_audio_delay_ms, &total_video_delay_ms));
478 // Trigger another call with the same values, delay should not be modified.
479 total_video_delay_ms = kDelayMs;
480 EXPECT_FALSE(DelayedStreams(kDelayMs, kDelayMs, current_audio_delay_ms,
481 &total_audio_delay_ms, &total_video_delay_ms));
482 // Change delay value, delay should not be modified.
483 const int kDelayMs2 = 5000;
484 current_audio_delay_ms = kDelayMs2;
485 total_video_delay_ms = kDelayMs2;
486 EXPECT_FALSE(DelayedStreams(kDelayMs2, kDelayMs2, current_audio_delay_ms,
487 &total_audio_delay_ms, &total_video_delay_ms));
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000488}
489
490TEST_F(StreamSynchronizationTest, BothDelayedAudioLaterWithBaseDelay) {
Åsa Persson0e578582020-02-03 11:13:20 +0100491 const int kBaseTargetDelayMs = 3000;
492 sync_.SetTargetBufferingDelay(kBaseTargetDelayMs);
493 BothDelayedAudioLaterTest(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000494}
495
496TEST_F(StreamSynchronizationTest, BothDelayedAudioClockDriftWithBaseDelay) {
Åsa Persson0e578582020-02-03 11:13:20 +0100497 const int kBaseTargetDelayMs = 3000;
498 sync_.SetTargetBufferingDelay(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000499 audio_clock_drift_ = 1.05;
Åsa Persson0e578582020-02-03 11:13:20 +0100500 BothDelayedAudioLaterTest(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000501}
502
503TEST_F(StreamSynchronizationTest, BothDelayedVideoClockDriftWithBaseDelay) {
Åsa Persson0e578582020-02-03 11:13:20 +0100504 const int kBaseTargetDelayMs = 3000;
505 sync_.SetTargetBufferingDelay(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000506 video_clock_drift_ = 1.05;
Åsa Persson0e578582020-02-03 11:13:20 +0100507 BothDelayedAudioLaterTest(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000508}
509
510TEST_F(StreamSynchronizationTest, BothDelayedVideoLaterWithBaseDelay) {
Åsa Persson0e578582020-02-03 11:13:20 +0100511 const int kBaseTargetDelayMs = 2000;
512 sync_.SetTargetBufferingDelay(kBaseTargetDelayMs);
513 BothDelayedVideoLaterTest(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000514}
515
516TEST_F(StreamSynchronizationTest,
517 BothDelayedVideoLaterAudioClockDriftWithBaseDelay) {
Åsa Persson0e578582020-02-03 11:13:20 +0100518 const int kBaseTargetDelayMs = 2000;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000519 audio_clock_drift_ = 1.05;
Åsa Persson0e578582020-02-03 11:13:20 +0100520 sync_.SetTargetBufferingDelay(kBaseTargetDelayMs);
521 BothDelayedVideoLaterTest(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000522}
523
524TEST_F(StreamSynchronizationTest,
525 BothDelayedVideoLaterVideoClockDriftWithBaseDelay) {
Åsa Persson0e578582020-02-03 11:13:20 +0100526 const int kBaseTargetDelayMs = 2000;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000527 video_clock_drift_ = 1.05;
Åsa Persson0e578582020-02-03 11:13:20 +0100528 sync_.SetTargetBufferingDelay(kBaseTargetDelayMs);
529 BothDelayedVideoLaterTest(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000530}
531
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000532} // namespace webrtc