blob: 5c6c79f6f10b25d88c6857e158e25a5208274605 [file] [log] [blame]
stefan@webrtc.org5f284982012-06-28 07:51:16 +00001/*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Jonas Olssona4d87372019-07-05 19:08:33 +020011#include "video/stream_synchronization.h"
12
kjellander@webrtc.org0fcaf992015-11-26 15:24:52 +010013#include <algorithm>
14
Åsa Persson8fe22fa2019-11-18 14:10:56 +010015#include "system_wrappers/include/clock.h"
Yves Gerey3e707812018-11-28 16:47:49 +010016#include "system_wrappers/include/ntp_time.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020017#include "test/gtest.h"
stefan@webrtc.org5f284982012-06-28 07:51:16 +000018
19namespace webrtc {
Åsa Persson8fe22fa2019-11-18 14:10:56 +010020namespace {
Åsa Persson0e578582020-02-03 11:13:20 +010021constexpr int kMaxChangeMs = 80; // From stream_synchronization.cc
Åsa Persson8fe22fa2019-11-18 14:10:56 +010022constexpr int kDefaultAudioFrequency = 8000;
23constexpr int kDefaultVideoFrequency = 90000;
24constexpr int kSmoothingFilter = 4 * 2;
25} // namespace
stefan@webrtc.org5f284982012-06-28 07:51:16 +000026
27class StreamSynchronizationTest : public ::testing::Test {
Åsa Persson8fe22fa2019-11-18 14:10:56 +010028 public:
29 StreamSynchronizationTest()
30 : sync_(0, 0), clock_sender_(98765000), clock_receiver_(43210000) {}
31
stefan@webrtc.org5f284982012-06-28 07:51:16 +000032 protected:
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +000033 // Generates the necessary RTCP measurements and RTP timestamps and computes
34 // the audio and video delays needed to get the two streams in sync.
Artem Titovab30d722021-07-27 16:22:11 +020035 // `audio_delay_ms` and `video_delay_ms` are the number of milliseconds after
Åsa Persson0e578582020-02-03 11:13:20 +010036 // capture which the frames are received.
Artem Titovab30d722021-07-27 16:22:11 +020037 // `current_audio_delay_ms` is the number of milliseconds which audio is
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +000038 // currently being delayed by the receiver.
39 bool DelayedStreams(int audio_delay_ms,
40 int video_delay_ms,
41 int current_audio_delay_ms,
Åsa Persson0e578582020-02-03 11:13:20 +010042 int* total_audio_delay_ms,
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +000043 int* total_video_delay_ms) {
Yves Gerey665174f2018-06-19 15:03:05 +020044 int audio_frequency =
45 static_cast<int>(kDefaultAudioFrequency * audio_clock_drift_ + 0.5);
Yves Gerey665174f2018-06-19 15:03:05 +020046 int video_frequency =
47 static_cast<int>(kDefaultVideoFrequency * video_clock_drift_ + 0.5);
Åsa Persson8fe22fa2019-11-18 14:10:56 +010048
49 // Generate NTP/RTP timestamp pair for both streams corresponding to RTCP.
asaperssonde9e5ff2016-11-02 07:14:03 -070050 bool new_sr;
stefan@webrtc.org5f284982012-06-28 07:51:16 +000051 StreamSynchronization::Measurements audio;
52 StreamSynchronization::Measurements video;
Åsa Persson8fe22fa2019-11-18 14:10:56 +010053 NtpTime ntp_time = clock_sender_.CurrentNtpTime();
asaperssonfe50b4d2016-12-22 07:53:51 -080054 uint32_t rtp_timestamp =
Åsa Persson8fe22fa2019-11-18 14:10:56 +010055 clock_sender_.CurrentTime().ms() * audio_frequency / 1000;
asaperssonfe50b4d2016-12-22 07:53:51 -080056 EXPECT_TRUE(audio.rtp_to_ntp.UpdateMeasurements(
57 ntp_time.seconds(), ntp_time.fractions(), rtp_timestamp, &new_sr));
Åsa Persson8fe22fa2019-11-18 14:10:56 +010058 clock_sender_.AdvanceTimeMilliseconds(100);
59 clock_receiver_.AdvanceTimeMilliseconds(100);
60 ntp_time = clock_sender_.CurrentNtpTime();
61 rtp_timestamp = clock_sender_.CurrentTime().ms() * video_frequency / 1000;
asaperssonfe50b4d2016-12-22 07:53:51 -080062 EXPECT_TRUE(video.rtp_to_ntp.UpdateMeasurements(
63 ntp_time.seconds(), ntp_time.fractions(), rtp_timestamp, &new_sr));
Åsa Persson8fe22fa2019-11-18 14:10:56 +010064 clock_sender_.AdvanceTimeMilliseconds(900);
65 clock_receiver_.AdvanceTimeMilliseconds(900);
66 ntp_time = clock_sender_.CurrentNtpTime();
67 rtp_timestamp = clock_sender_.CurrentTime().ms() * audio_frequency / 1000;
asaperssonfe50b4d2016-12-22 07:53:51 -080068 EXPECT_TRUE(audio.rtp_to_ntp.UpdateMeasurements(
69 ntp_time.seconds(), ntp_time.fractions(), rtp_timestamp, &new_sr));
Åsa Persson8fe22fa2019-11-18 14:10:56 +010070 clock_sender_.AdvanceTimeMilliseconds(100);
71 clock_receiver_.AdvanceTimeMilliseconds(100);
72 ntp_time = clock_sender_.CurrentNtpTime();
73 rtp_timestamp = clock_sender_.CurrentTime().ms() * video_frequency / 1000;
asaperssonfe50b4d2016-12-22 07:53:51 -080074 EXPECT_TRUE(video.rtp_to_ntp.UpdateMeasurements(
75 ntp_time.seconds(), ntp_time.fractions(), rtp_timestamp, &new_sr));
Åsa Persson8fe22fa2019-11-18 14:10:56 +010076 clock_sender_.AdvanceTimeMilliseconds(900);
77 clock_receiver_.AdvanceTimeMilliseconds(900);
stefan@webrtc.org5f284982012-06-28 07:51:16 +000078
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +000079 // Capture an audio and a video frame at the same time.
asaperssonb7e7b492016-11-17 02:27:14 -080080 audio.latest_timestamp =
Åsa Persson8fe22fa2019-11-18 14:10:56 +010081 clock_sender_.CurrentTime().ms() * audio_frequency / 1000;
asaperssonb7e7b492016-11-17 02:27:14 -080082 video.latest_timestamp =
Åsa Persson8fe22fa2019-11-18 14:10:56 +010083 clock_sender_.CurrentTime().ms() * video_frequency / 1000;
stefan@webrtc.org5f284982012-06-28 07:51:16 +000084
85 if (audio_delay_ms > video_delay_ms) {
86 // Audio later than video.
Åsa Persson8fe22fa2019-11-18 14:10:56 +010087 clock_receiver_.AdvanceTimeMilliseconds(video_delay_ms);
88 video.latest_receive_time_ms = clock_receiver_.CurrentTime().ms();
89 clock_receiver_.AdvanceTimeMilliseconds(audio_delay_ms - video_delay_ms);
90 audio.latest_receive_time_ms = clock_receiver_.CurrentTime().ms();
stefan@webrtc.org5f284982012-06-28 07:51:16 +000091 } else {
92 // Video later than audio.
Åsa Persson8fe22fa2019-11-18 14:10:56 +010093 clock_receiver_.AdvanceTimeMilliseconds(audio_delay_ms);
94 audio.latest_receive_time_ms = clock_receiver_.CurrentTime().ms();
95 clock_receiver_.AdvanceTimeMilliseconds(video_delay_ms - audio_delay_ms);
96 video.latest_receive_time_ms = clock_receiver_.CurrentTime().ms();
stefan@webrtc.org5f284982012-06-28 07:51:16 +000097 }
Åsa Persson0e578582020-02-03 11:13:20 +010098
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +000099 int relative_delay_ms;
Åsa Persson0e578582020-02-03 11:13:20 +0100100 EXPECT_TRUE(StreamSynchronization::ComputeRelativeDelay(
101 audio, video, &relative_delay_ms));
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000102 EXPECT_EQ(video_delay_ms - audio_delay_ms, relative_delay_ms);
Åsa Persson0e578582020-02-03 11:13:20 +0100103
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100104 return sync_.ComputeDelays(relative_delay_ms, current_audio_delay_ms,
Åsa Persson0e578582020-02-03 11:13:20 +0100105 total_audio_delay_ms, total_video_delay_ms);
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000106 }
107
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000108 // Simulate audio playback 300 ms after capture and video rendering 100 ms
109 // after capture. Verify that the correct extra delays are calculated for
110 // audio and video, and that they change correctly when we simulate that
111 // NetEQ or the VCM adds more delay to the streams.
Åsa Persson0e578582020-02-03 11:13:20 +0100112 void BothDelayedAudioLaterTest(int base_target_delay_ms) {
113 const int kAudioDelayMs = base_target_delay_ms + 300;
114 const int kVideoDelayMs = base_target_delay_ms + 100;
115 int current_audio_delay_ms = base_target_delay_ms;
116 int total_audio_delay_ms = 0;
117 int total_video_delay_ms = base_target_delay_ms;
118 int filtered_move = (kAudioDelayMs - kVideoDelayMs) / kSmoothingFilter;
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000119
Åsa Persson0e578582020-02-03 11:13:20 +0100120 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
121 current_audio_delay_ms, &total_audio_delay_ms,
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000122 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 11:13:20 +0100123 EXPECT_EQ(base_target_delay_ms + filtered_move, total_video_delay_ms);
124 EXPECT_EQ(base_target_delay_ms, total_audio_delay_ms);
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000125
Åsa Persson0e578582020-02-03 11:13:20 +0100126 // Set new current delay.
127 current_audio_delay_ms = total_audio_delay_ms;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100128 clock_sender_.AdvanceTimeMilliseconds(1000);
129 clock_receiver_.AdvanceTimeMilliseconds(
Åsa Persson0e578582020-02-03 11:13:20 +0100130 1000 - std::max(kAudioDelayMs, kVideoDelayMs));
131 // Simulate base_target_delay_ms minimum delay in the VCM.
132 total_video_delay_ms = base_target_delay_ms;
133 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
134 current_audio_delay_ms, &total_audio_delay_ms,
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000135 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 11:13:20 +0100136 EXPECT_EQ(base_target_delay_ms + 2 * filtered_move, total_video_delay_ms);
137 EXPECT_EQ(base_target_delay_ms, total_audio_delay_ms);
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000138
Åsa Persson0e578582020-02-03 11:13:20 +0100139 // Set new current delay.
140 current_audio_delay_ms = total_audio_delay_ms;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100141 clock_sender_.AdvanceTimeMilliseconds(1000);
142 clock_receiver_.AdvanceTimeMilliseconds(
Åsa Persson0e578582020-02-03 11:13:20 +0100143 1000 - std::max(kAudioDelayMs, kVideoDelayMs));
144 // Simulate base_target_delay_ms minimum delay in the VCM.
145 total_video_delay_ms = base_target_delay_ms;
146 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
147 current_audio_delay_ms, &total_audio_delay_ms,
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000148 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 11:13:20 +0100149 EXPECT_EQ(base_target_delay_ms + 3 * filtered_move, total_video_delay_ms);
150 EXPECT_EQ(base_target_delay_ms, total_audio_delay_ms);
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000151
152 // Simulate that NetEQ introduces some audio delay.
Åsa Persson0e578582020-02-03 11:13:20 +0100153 const int kNeteqDelayIncrease = 50;
154 current_audio_delay_ms = base_target_delay_ms + kNeteqDelayIncrease;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100155 clock_sender_.AdvanceTimeMilliseconds(1000);
156 clock_receiver_.AdvanceTimeMilliseconds(
Åsa Persson0e578582020-02-03 11:13:20 +0100157 1000 - std::max(kAudioDelayMs, kVideoDelayMs));
158 // Simulate base_target_delay_ms minimum delay in the VCM.
159 total_video_delay_ms = base_target_delay_ms;
160 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
161 current_audio_delay_ms, &total_audio_delay_ms,
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000162 &total_video_delay_ms));
pwestin@webrtc.org63117332013-04-22 18:57:14 +0000163 filtered_move = 3 * filtered_move +
Åsa Persson0e578582020-02-03 11:13:20 +0100164 (kNeteqDelayIncrease + kAudioDelayMs - kVideoDelayMs) /
Yves Gerey665174f2018-06-19 15:03:05 +0200165 kSmoothingFilter;
Åsa Persson0e578582020-02-03 11:13:20 +0100166 EXPECT_EQ(base_target_delay_ms + filtered_move, total_video_delay_ms);
167 EXPECT_EQ(base_target_delay_ms, total_audio_delay_ms);
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000168
169 // Simulate that NetEQ reduces its delay.
Åsa Persson0e578582020-02-03 11:13:20 +0100170 const int kNeteqDelayDecrease = 10;
171 current_audio_delay_ms = base_target_delay_ms + kNeteqDelayDecrease;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100172 clock_sender_.AdvanceTimeMilliseconds(1000);
173 clock_receiver_.AdvanceTimeMilliseconds(
Åsa Persson0e578582020-02-03 11:13:20 +0100174 1000 - std::max(kAudioDelayMs, kVideoDelayMs));
175 // Simulate base_target_delay_ms minimum delay in the VCM.
176 total_video_delay_ms = base_target_delay_ms;
177 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
178 current_audio_delay_ms, &total_audio_delay_ms,
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000179 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 11:13:20 +0100180 filtered_move =
181 filtered_move + (kNeteqDelayDecrease + kAudioDelayMs - kVideoDelayMs) /
182 kSmoothingFilter;
183 EXPECT_EQ(base_target_delay_ms + filtered_move, total_video_delay_ms);
184 EXPECT_EQ(base_target_delay_ms, total_audio_delay_ms);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000185 }
186
Åsa Persson0e578582020-02-03 11:13:20 +0100187 void BothDelayedVideoLaterTest(int base_target_delay_ms) {
188 const int kAudioDelayMs = base_target_delay_ms + 100;
189 const int kVideoDelayMs = base_target_delay_ms + 300;
190 int current_audio_delay_ms = base_target_delay_ms;
191 int total_audio_delay_ms = 0;
192 int total_video_delay_ms = base_target_delay_ms;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000193
Åsa Persson0e578582020-02-03 11:13:20 +0100194 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
195 current_audio_delay_ms, &total_audio_delay_ms,
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000196 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 11:13:20 +0100197 EXPECT_EQ(base_target_delay_ms, total_video_delay_ms);
198 // The audio delay is not allowed to change more than this.
199 EXPECT_GE(base_target_delay_ms + kMaxChangeMs, total_audio_delay_ms);
200 int last_total_audio_delay_ms = total_audio_delay_ms;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000201
Åsa Persson0e578582020-02-03 11:13:20 +0100202 // Set new current audio delay.
203 current_audio_delay_ms = total_audio_delay_ms;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100204 clock_sender_.AdvanceTimeMilliseconds(1000);
205 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 11:13:20 +0100206 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
207 current_audio_delay_ms, &total_audio_delay_ms,
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000208 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 11:13:20 +0100209 EXPECT_EQ(base_target_delay_ms, total_video_delay_ms);
210 EXPECT_EQ(last_total_audio_delay_ms +
211 MaxAudioDelayChangeMs(
Yves Gerey665174f2018-06-19 15:03:05 +0200212 current_audio_delay_ms,
Åsa Persson0e578582020-02-03 11:13:20 +0100213 base_target_delay_ms + kVideoDelayMs - kAudioDelayMs),
214 total_audio_delay_ms);
215 last_total_audio_delay_ms = total_audio_delay_ms;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000216
Åsa Persson0e578582020-02-03 11:13:20 +0100217 // Set new current audio delay.
218 current_audio_delay_ms = total_audio_delay_ms;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100219 clock_sender_.AdvanceTimeMilliseconds(1000);
220 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 11:13:20 +0100221 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
222 current_audio_delay_ms, &total_audio_delay_ms,
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000223 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 11:13:20 +0100224 EXPECT_EQ(base_target_delay_ms, total_video_delay_ms);
225 EXPECT_EQ(last_total_audio_delay_ms +
226 MaxAudioDelayChangeMs(
Yves Gerey665174f2018-06-19 15:03:05 +0200227 current_audio_delay_ms,
Åsa Persson0e578582020-02-03 11:13:20 +0100228 base_target_delay_ms + kVideoDelayMs - kAudioDelayMs),
229 total_audio_delay_ms);
230 last_total_audio_delay_ms = total_audio_delay_ms;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000231
232 // Simulate that NetEQ for some reason reduced the delay.
Åsa Persson0e578582020-02-03 11:13:20 +0100233 current_audio_delay_ms = base_target_delay_ms + 10;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100234 clock_sender_.AdvanceTimeMilliseconds(1000);
235 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 11:13:20 +0100236 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
237 current_audio_delay_ms, &total_audio_delay_ms,
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000238 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 11:13:20 +0100239 EXPECT_EQ(base_target_delay_ms, total_video_delay_ms);
240 EXPECT_EQ(last_total_audio_delay_ms +
241 MaxAudioDelayChangeMs(
Yves Gerey665174f2018-06-19 15:03:05 +0200242 current_audio_delay_ms,
Åsa Persson0e578582020-02-03 11:13:20 +0100243 base_target_delay_ms + kVideoDelayMs - kAudioDelayMs),
244 total_audio_delay_ms);
245 last_total_audio_delay_ms = total_audio_delay_ms;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000246
247 // Simulate that NetEQ for some reason significantly increased the delay.
Åsa Persson0e578582020-02-03 11:13:20 +0100248 current_audio_delay_ms = base_target_delay_ms + 350;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100249 clock_sender_.AdvanceTimeMilliseconds(1000);
250 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 11:13:20 +0100251 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, kVideoDelayMs,
252 current_audio_delay_ms, &total_audio_delay_ms,
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000253 &total_video_delay_ms));
Åsa Persson0e578582020-02-03 11:13:20 +0100254 EXPECT_EQ(base_target_delay_ms, total_video_delay_ms);
255 EXPECT_EQ(last_total_audio_delay_ms +
256 MaxAudioDelayChangeMs(
Yves Gerey665174f2018-06-19 15:03:05 +0200257 current_audio_delay_ms,
Åsa Persson0e578582020-02-03 11:13:20 +0100258 base_target_delay_ms + kVideoDelayMs - kAudioDelayMs),
259 total_audio_delay_ms);
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000260 }
261
Åsa Persson0e578582020-02-03 11:13:20 +0100262 int MaxAudioDelayChangeMs(int current_audio_delay_ms, int delay_ms) const {
263 int diff_ms = (delay_ms - current_audio_delay_ms) / kSmoothingFilter;
264 diff_ms = std::min(diff_ms, kMaxChangeMs);
265 diff_ms = std::max(diff_ms, -kMaxChangeMs);
266 return diff_ms;
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000267 }
268
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100269 StreamSynchronization sync_;
270 SimulatedClock clock_sender_;
271 SimulatedClock clock_receiver_;
272 double audio_clock_drift_ = 1.0;
273 double video_clock_drift_ = 1.0;
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000274};
275
276TEST_F(StreamSynchronizationTest, NoDelay) {
Åsa Persson0e578582020-02-03 11:13:20 +0100277 int total_audio_delay_ms = 0;
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000278 int total_video_delay_ms = 0;
279
Åsa Persson0e578582020-02-03 11:13:20 +0100280 EXPECT_FALSE(DelayedStreams(/*audio_delay_ms=*/0, /*video_delay_ms=*/0,
281 /*current_audio_delay_ms=*/0,
282 &total_audio_delay_ms, &total_video_delay_ms));
283 EXPECT_EQ(0, total_audio_delay_ms);
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000284 EXPECT_EQ(0, total_video_delay_ms);
285}
286
Åsa Persson0e578582020-02-03 11:13:20 +0100287TEST_F(StreamSynchronizationTest, VideoDelayed) {
288 const int kAudioDelayMs = 200;
289 int total_audio_delay_ms = 0;
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000290 int total_video_delay_ms = 0;
291
Åsa Persson0e578582020-02-03 11:13:20 +0100292 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, /*video_delay_ms=*/0,
293 /*current_audio_delay_ms=*/0,
294 &total_audio_delay_ms, &total_video_delay_ms));
295 EXPECT_EQ(0, total_audio_delay_ms);
296 // The delay is not allowed to change more than this.
297 EXPECT_EQ(kAudioDelayMs / kSmoothingFilter, total_video_delay_ms);
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000298
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000299 // Simulate 0 minimum delay in the VCM.
300 total_video_delay_ms = 0;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100301 clock_sender_.AdvanceTimeMilliseconds(1000);
302 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 11:13:20 +0100303 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, /*video_delay_ms=*/0,
304 /*current_audio_delay_ms=*/0,
305 &total_audio_delay_ms, &total_video_delay_ms));
306 EXPECT_EQ(0, total_audio_delay_ms);
307 EXPECT_EQ(2 * kAudioDelayMs / kSmoothingFilter, total_video_delay_ms);
308
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000309 // Simulate 0 minimum delay in the VCM.
310 total_video_delay_ms = 0;
Åsa Persson0e578582020-02-03 11:13:20 +0100311 clock_sender_.AdvanceTimeMilliseconds(1000);
312 clock_receiver_.AdvanceTimeMilliseconds(800);
313 EXPECT_TRUE(DelayedStreams(kAudioDelayMs, /*video_delay_ms=*/0,
314 /*current_audio_delay_ms=*/0,
315 &total_audio_delay_ms, &total_video_delay_ms));
316 EXPECT_EQ(0, total_audio_delay_ms);
317 EXPECT_EQ(3 * kAudioDelayMs / kSmoothingFilter, total_video_delay_ms);
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000318}
319
Åsa Persson0e578582020-02-03 11:13:20 +0100320TEST_F(StreamSynchronizationTest, AudioDelayed) {
321 const int kVideoDelayMs = 200;
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000322 int current_audio_delay_ms = 0;
Åsa Persson0e578582020-02-03 11:13:20 +0100323 int total_audio_delay_ms = 0;
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000324 int total_video_delay_ms = 0;
325
Åsa Persson0e578582020-02-03 11:13:20 +0100326 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
327 current_audio_delay_ms, &total_audio_delay_ms,
328 &total_video_delay_ms));
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000329 EXPECT_EQ(0, total_video_delay_ms);
Åsa Persson0e578582020-02-03 11:13:20 +0100330 // The delay is not allowed to change more than this.
331 EXPECT_EQ(kVideoDelayMs / kSmoothingFilter, total_audio_delay_ms);
332 int last_total_audio_delay_ms = total_audio_delay_ms;
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000333
Åsa Persson0e578582020-02-03 11:13:20 +0100334 // Set new current audio delay.
335 current_audio_delay_ms = total_audio_delay_ms;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100336 clock_sender_.AdvanceTimeMilliseconds(1000);
337 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 11:13:20 +0100338 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
339 current_audio_delay_ms, &total_audio_delay_ms,
340 &total_video_delay_ms));
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000341 EXPECT_EQ(0, total_video_delay_ms);
Åsa Persson0e578582020-02-03 11:13:20 +0100342 EXPECT_EQ(last_total_audio_delay_ms +
343 MaxAudioDelayChangeMs(current_audio_delay_ms, kVideoDelayMs),
344 total_audio_delay_ms);
345 last_total_audio_delay_ms = total_audio_delay_ms;
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000346
Åsa Persson0e578582020-02-03 11:13:20 +0100347 // Set new current audio delay.
348 current_audio_delay_ms = total_audio_delay_ms;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100349 clock_sender_.AdvanceTimeMilliseconds(1000);
350 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 11:13:20 +0100351 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
352 current_audio_delay_ms, &total_audio_delay_ms,
353 &total_video_delay_ms));
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000354 EXPECT_EQ(0, total_video_delay_ms);
Åsa Persson0e578582020-02-03 11:13:20 +0100355 EXPECT_EQ(last_total_audio_delay_ms +
356 MaxAudioDelayChangeMs(current_audio_delay_ms, kVideoDelayMs),
357 total_audio_delay_ms);
358 last_total_audio_delay_ms = total_audio_delay_ms;
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000359
360 // Simulate that NetEQ for some reason reduced the delay.
pwestin@webrtc.org63117332013-04-22 18:57:14 +0000361 current_audio_delay_ms = 10;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100362 clock_sender_.AdvanceTimeMilliseconds(1000);
363 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 11:13:20 +0100364 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
365 current_audio_delay_ms, &total_audio_delay_ms,
366 &total_video_delay_ms));
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000367 EXPECT_EQ(0, total_video_delay_ms);
Åsa Persson0e578582020-02-03 11:13:20 +0100368 EXPECT_EQ(last_total_audio_delay_ms +
369 MaxAudioDelayChangeMs(current_audio_delay_ms, kVideoDelayMs),
370 total_audio_delay_ms);
371 last_total_audio_delay_ms = total_audio_delay_ms;
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000372
373 // Simulate that NetEQ for some reason significantly increased the delay.
pwestin@webrtc.org63117332013-04-22 18:57:14 +0000374 current_audio_delay_ms = 350;
Åsa Persson8fe22fa2019-11-18 14:10:56 +0100375 clock_sender_.AdvanceTimeMilliseconds(1000);
376 clock_receiver_.AdvanceTimeMilliseconds(800);
Åsa Persson0e578582020-02-03 11:13:20 +0100377 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
378 current_audio_delay_ms, &total_audio_delay_ms,
379 &total_video_delay_ms));
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000380 EXPECT_EQ(0, total_video_delay_ms);
Åsa Persson0e578582020-02-03 11:13:20 +0100381 EXPECT_EQ(last_total_audio_delay_ms +
382 MaxAudioDelayChangeMs(current_audio_delay_ms, kVideoDelayMs),
383 total_audio_delay_ms);
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000384}
385
Shyam Sadhwani986e7452020-09-14 10:12:54 -0700386TEST_F(StreamSynchronizationTest, NoAudioIncomingUnboundedIncrease) {
387 // Test how audio delay can grow unbounded when audio stops coming in.
388 // This is handled in caller of RtpStreamsSynchronizer, for example in
389 // RtpStreamsSynchronizer by not updating delays when audio samples stop
390 // coming in.
391 const int kVideoDelayMs = 300;
392 const int kAudioDelayMs = 100;
393 int current_audio_delay_ms = kAudioDelayMs;
394 int total_audio_delay_ms = 0;
395 int total_video_delay_ms = 0;
396
397 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
398 current_audio_delay_ms, &total_audio_delay_ms,
399 &total_video_delay_ms));
400 EXPECT_EQ(0, total_video_delay_ms);
401 // The delay is not allowed to change more than this.
402 EXPECT_EQ((kVideoDelayMs - kAudioDelayMs) / kSmoothingFilter,
403 total_audio_delay_ms);
404 int last_total_audio_delay_ms = total_audio_delay_ms;
405
406 // Set new current audio delay: simulate audio samples are flowing in.
407 current_audio_delay_ms = total_audio_delay_ms;
408
409 clock_sender_.AdvanceTimeMilliseconds(1000);
410 clock_receiver_.AdvanceTimeMilliseconds(1000);
411 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
412 current_audio_delay_ms, &total_audio_delay_ms,
413 &total_video_delay_ms));
414 EXPECT_EQ(0, total_video_delay_ms);
415 EXPECT_EQ(last_total_audio_delay_ms +
416 MaxAudioDelayChangeMs(current_audio_delay_ms, kVideoDelayMs),
417 total_audio_delay_ms);
418 last_total_audio_delay_ms = total_audio_delay_ms;
419
420 // Simulate no incoming audio by not update audio delay.
421 const int kSimulationSecs = 300; // 5min
422 const int kMaxDeltaDelayMs = 10000; // max delay for audio in webrtc
423 for (auto time_secs = 0; time_secs < kSimulationSecs; time_secs++) {
424 clock_sender_.AdvanceTimeMilliseconds(1000);
425 clock_receiver_.AdvanceTimeMilliseconds(1000);
426 EXPECT_TRUE(DelayedStreams(/*audio_delay_ms=*/0, kVideoDelayMs,
427 current_audio_delay_ms, &total_audio_delay_ms,
428 &total_video_delay_ms));
429 EXPECT_EQ(0, total_video_delay_ms);
430
431 // Audio delay does not go above kMaxDeltaDelayMs.
432 EXPECT_EQ(std::min(kMaxDeltaDelayMs,
433 last_total_audio_delay_ms +
434 MaxAudioDelayChangeMs(current_audio_delay_ms,
435 kVideoDelayMs)),
436 total_audio_delay_ms);
437 last_total_audio_delay_ms = total_audio_delay_ms;
438 }
439 // By now the audio delay has grown unbounded to kMaxDeltaDelayMs.
440 EXPECT_EQ(kMaxDeltaDelayMs, last_total_audio_delay_ms);
441}
442
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000443TEST_F(StreamSynchronizationTest, BothDelayedVideoLater) {
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000444 BothDelayedVideoLaterTest(0);
445}
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000446
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000447TEST_F(StreamSynchronizationTest, BothDelayedVideoLaterAudioClockDrift) {
448 audio_clock_drift_ = 1.05;
449 BothDelayedVideoLaterTest(0);
450}
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000451
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000452TEST_F(StreamSynchronizationTest, BothDelayedVideoLaterVideoClockDrift) {
453 video_clock_drift_ = 1.05;
454 BothDelayedVideoLaterTest(0);
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000455}
456
457TEST_F(StreamSynchronizationTest, BothDelayedAudioLater) {
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000458 BothDelayedAudioLaterTest(0);
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000459}
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000460
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000461TEST_F(StreamSynchronizationTest, BothDelayedAudioClockDrift) {
462 audio_clock_drift_ = 1.05;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000463 BothDelayedAudioLaterTest(0);
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000464}
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000465
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000466TEST_F(StreamSynchronizationTest, BothDelayedVideoClockDrift) {
467 video_clock_drift_ = 1.05;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000468 BothDelayedAudioLaterTest(0);
stefan@webrtc.org7c3523c2012-09-11 07:00:42 +0000469}
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000470
Åsa Persson0e578582020-02-03 11:13:20 +0100471TEST_F(StreamSynchronizationTest, BothEquallyDelayed) {
472 const int kDelayMs = 2000;
473 int current_audio_delay_ms = kDelayMs;
474 int total_audio_delay_ms = 0;
475 int total_video_delay_ms = kDelayMs;
476 // In sync, expect no change.
477 EXPECT_FALSE(DelayedStreams(kDelayMs, kDelayMs, current_audio_delay_ms,
478 &total_audio_delay_ms, &total_video_delay_ms));
479 // Trigger another call with the same values, delay should not be modified.
480 total_video_delay_ms = kDelayMs;
481 EXPECT_FALSE(DelayedStreams(kDelayMs, kDelayMs, current_audio_delay_ms,
482 &total_audio_delay_ms, &total_video_delay_ms));
483 // Change delay value, delay should not be modified.
484 const int kDelayMs2 = 5000;
485 current_audio_delay_ms = kDelayMs2;
486 total_video_delay_ms = kDelayMs2;
487 EXPECT_FALSE(DelayedStreams(kDelayMs2, kDelayMs2, current_audio_delay_ms,
488 &total_audio_delay_ms, &total_video_delay_ms));
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000489}
490
491TEST_F(StreamSynchronizationTest, BothDelayedAudioLaterWithBaseDelay) {
Åsa Persson0e578582020-02-03 11:13:20 +0100492 const int kBaseTargetDelayMs = 3000;
493 sync_.SetTargetBufferingDelay(kBaseTargetDelayMs);
494 BothDelayedAudioLaterTest(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000495}
496
497TEST_F(StreamSynchronizationTest, BothDelayedAudioClockDriftWithBaseDelay) {
Åsa Persson0e578582020-02-03 11:13:20 +0100498 const int kBaseTargetDelayMs = 3000;
499 sync_.SetTargetBufferingDelay(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000500 audio_clock_drift_ = 1.05;
Åsa Persson0e578582020-02-03 11:13:20 +0100501 BothDelayedAudioLaterTest(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000502}
503
504TEST_F(StreamSynchronizationTest, BothDelayedVideoClockDriftWithBaseDelay) {
Åsa Persson0e578582020-02-03 11:13:20 +0100505 const int kBaseTargetDelayMs = 3000;
506 sync_.SetTargetBufferingDelay(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000507 video_clock_drift_ = 1.05;
Åsa Persson0e578582020-02-03 11:13:20 +0100508 BothDelayedAudioLaterTest(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000509}
510
511TEST_F(StreamSynchronizationTest, BothDelayedVideoLaterWithBaseDelay) {
Åsa Persson0e578582020-02-03 11:13:20 +0100512 const int kBaseTargetDelayMs = 2000;
513 sync_.SetTargetBufferingDelay(kBaseTargetDelayMs);
514 BothDelayedVideoLaterTest(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000515}
516
517TEST_F(StreamSynchronizationTest,
518 BothDelayedVideoLaterAudioClockDriftWithBaseDelay) {
Åsa Persson0e578582020-02-03 11:13:20 +0100519 const int kBaseTargetDelayMs = 2000;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000520 audio_clock_drift_ = 1.05;
Åsa Persson0e578582020-02-03 11:13:20 +0100521 sync_.SetTargetBufferingDelay(kBaseTargetDelayMs);
522 BothDelayedVideoLaterTest(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000523}
524
525TEST_F(StreamSynchronizationTest,
526 BothDelayedVideoLaterVideoClockDriftWithBaseDelay) {
Åsa Persson0e578582020-02-03 11:13:20 +0100527 const int kBaseTargetDelayMs = 2000;
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000528 video_clock_drift_ = 1.05;
Åsa Persson0e578582020-02-03 11:13:20 +0100529 sync_.SetTargetBufferingDelay(kBaseTargetDelayMs);
530 BothDelayedVideoLaterTest(kBaseTargetDelayMs);
mikhal@webrtc.orgef9f76a2013-02-15 23:22:18 +0000531}
532
stefan@webrtc.org5f284982012-06-28 07:51:16 +0000533} // namespace webrtc