blob: 458944aefa555c29454bd1942320227cb38560f3 [file] [log] [blame]
Tommiae4d0972020-05-18 08:45:38 +02001/*
2 * Copyright 2017 The WebRTC Project Authors. All rights reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "video/video_receive_stream2.h"
12
13#include <algorithm>
Evan Shrubsolea4062722022-05-02 17:46:07 +020014#include <cstddef>
Evan Shrubsole7cbd8de2022-08-16 08:08:53 +000015#include <deque>
Markus Handell588f9b32021-04-08 19:19:50 +020016#include <limits>
Tommiae4d0972020-05-18 08:45:38 +020017#include <memory>
Evan Shrubsole1c184772022-04-26 09:47:49 +020018#include <ostream>
Evan Shrubsole7cbd8de2022-08-16 08:08:53 +000019#include <queue>
Evan Shrubsole9a999052021-12-12 15:27:00 +010020#include <tuple>
Tommiae4d0972020-05-18 08:45:38 +020021#include <utility>
22#include <vector>
23
Evan Shrubsole6dbc1722022-03-22 12:20:11 +010024#include "absl/memory/memory.h"
Evan Shrubsole44be5792022-04-26 16:24:41 +020025#include "absl/types/optional.h"
Evan Shrubsolea4062722022-05-02 17:46:07 +020026#include "api/metronome/test/fake_metronome.h"
Danil Chapovalov9cd4d492021-08-03 14:59:00 +020027#include "api/test/mock_video_decoder.h"
28#include "api/test/mock_video_decoder_factory.h"
Evan Shrubsole44be5792022-04-26 16:24:41 +020029#include "api/test/time_controller.h"
Evan Shrubsolea4062722022-05-02 17:46:07 +020030#include "api/units/frequency.h"
Evan Shrubsole44be5792022-04-26 16:24:41 +020031#include "api/units/time_delta.h"
Evan Shrubsolea4062722022-05-02 17:46:07 +020032#include "api/video/encoded_image.h"
Evan Shrubsole44be5792022-04-26 16:24:41 +020033#include "api/video/recordable_encoded_frame.h"
34#include "api/video/test/video_frame_matchers.h"
Markus Handell588f9b32021-04-08 19:19:50 +020035#include "api/video/video_frame.h"
Evan Shrubsole1c184772022-04-26 09:47:49 +020036#include "api/video_codecs/sdp_video_format.h"
Tommiae4d0972020-05-18 08:45:38 +020037#include "api/video_codecs/video_decoder.h"
38#include "call/rtp_stream_receiver_controller.h"
Evan Shrubsole1c184772022-04-26 09:47:49 +020039#include "call/video_receive_stream.h"
Tommiae4d0972020-05-18 08:45:38 +020040#include "common_video/test/utilities.h"
Tommi90738dd2021-05-31 17:36:47 +020041#include "media/engine/fake_webrtc_call.h"
Tommiae4d0972020-05-18 08:45:38 +020042#include "modules/pacing/packet_router.h"
43#include "modules/rtp_rtcp/source/rtp_packet_to_send.h"
Tommiae4d0972020-05-18 08:45:38 +020044#include "modules/video_coding/encoded_frame.h"
Evan Shrubsoledcb9c5d2022-06-13 15:39:53 +000045#include "rtc_base/logging.h"
Tommiae4d0972020-05-18 08:45:38 +020046#include "system_wrappers/include/clock.h"
47#include "test/fake_decoder.h"
Evan Shrubsolea0ee64c2022-04-26 10:09:04 +020048#include "test/fake_encoded_frame.h"
Tommiae4d0972020-05-18 08:45:38 +020049#include "test/gmock.h"
50#include "test/gtest.h"
Danil Chapovalov9cd4d492021-08-03 14:59:00 +020051#include "test/mock_transport.h"
philipel27b35a72022-07-05 09:59:55 +020052#include "test/rtcp_packet_parser.h"
Tommiae4d0972020-05-18 08:45:38 +020053#include "test/time_controller/simulated_time_controller.h"
54#include "test/video_decoder_proxy_factory.h"
55#include "video/call_stats2.h"
56
57namespace webrtc {
Evan Shrubsole1c184772022-04-26 09:47:49 +020058
59// Printing SdpVideoFormat for gmock argument matchers.
60void PrintTo(const SdpVideoFormat& value, std::ostream* os) {
61 *os << value.ToString();
62}
63
Evan Shrubsole44be5792022-04-26 16:24:41 +020064void PrintTo(const RecordableEncodedFrame::EncodedResolution& value,
65 std::ostream* os) {
66 *os << value.width << "x" << value.height;
67}
68
69void PrintTo(const RecordableEncodedFrame& value, std::ostream* os) {
70 *os << "RecordableEncodedFrame(render_time=" << value.render_time()
71 << " resolution=" << ::testing::PrintToString(value.resolution()) << ")";
72}
73
Evan Shrubsolea4062722022-05-02 17:46:07 +020074} // namespace webrtc
75
76namespace webrtc {
77
Tommiae4d0972020-05-18 08:45:38 +020078namespace {
79
Evan Shrubsole44be5792022-04-26 16:24:41 +020080using test::video_frame_matchers::NtpTimestamp;
81using test::video_frame_matchers::PacketInfos;
82using test::video_frame_matchers::Rotation;
Tommiae4d0972020-05-18 08:45:38 +020083using ::testing::_;
Markus Handell588f9b32021-04-08 19:19:50 +020084using ::testing::AllOf;
Evan Shrubsoledcb9c5d2022-06-13 15:39:53 +000085using ::testing::AnyNumber;
Tommiae4d0972020-05-18 08:45:38 +020086using ::testing::ElementsAreArray;
Evan Shrubsole44be5792022-04-26 16:24:41 +020087using ::testing::Eq;
Markus Handell588f9b32021-04-08 19:19:50 +020088using ::testing::Field;
89using ::testing::InSequence;
Tommiae4d0972020-05-18 08:45:38 +020090using ::testing::Invoke;
91using ::testing::IsEmpty;
Johannes Kronbbf639e2022-06-15 12:27:23 +020092using ::testing::Optional;
Evan Shrubsolea4062722022-05-02 17:46:07 +020093using ::testing::Pointee;
Markus Handell588f9b32021-04-08 19:19:50 +020094using ::testing::Property;
Evan Shrubsole44be5792022-04-26 16:24:41 +020095using ::testing::Return;
Tommiae4d0972020-05-18 08:45:38 +020096using ::testing::SizeIs;
Danil Chapovalovd08930d2021-08-12 13:26:55 +020097using ::testing::WithoutArgs;
Tommiae4d0972020-05-18 08:45:38 +020098
Evan Shrubsole44be5792022-04-26 16:24:41 +020099auto RenderedFrameWith(::testing::Matcher<VideoFrame> m) {
100 return Optional(m);
101}
Evan Shrubsolea4062722022-05-02 17:46:07 +0200102auto RenderedFrame() {
103 return RenderedFrameWith(_);
104}
105testing::Matcher<absl::optional<VideoFrame>> DidNotReceiveFrame() {
106 return Eq(absl::nullopt);
107}
Evan Shrubsole44be5792022-04-26 16:24:41 +0200108
109constexpr TimeDelta kDefaultTimeOut = TimeDelta::Millis(50);
Evan Shrubsole14ee8032022-04-26 16:01:44 +0200110constexpr int kDefaultNumCpuCores = 2;
Tommiae4d0972020-05-18 08:45:38 +0200111
Evan Shrubsole44be5792022-04-26 16:24:41 +0200112constexpr Timestamp kStartTime = Timestamp::Millis(1'337'000);
Evan Shrubsolea4062722022-05-02 17:46:07 +0200113constexpr Frequency k30Fps = Frequency::Hertz(30);
114constexpr TimeDelta k30FpsDelay = 1 / k30Fps;
115constexpr Frequency kRtpTimestampHz = Frequency::KiloHertz(90);
116constexpr uint32_t k30FpsRtpTimestampDelta = kRtpTimestampHz / k30Fps;
117constexpr uint32_t kFirstRtpTimestamp = 90000;
Evan Shrubsole44be5792022-04-26 16:24:41 +0200118
Evan Shrubsoledcb9c5d2022-06-13 15:39:53 +0000119class FakeVideoRenderer : public rtc::VideoSinkInterface<VideoFrame> {
Evan Shrubsole44be5792022-04-26 16:24:41 +0200120 public:
Evan Shrubsole7cbd8de2022-08-16 08:08:53 +0000121 explicit FakeVideoRenderer(TimeController* time_controller)
122 : time_controller_(time_controller) {}
Evan Shrubsole44be5792022-04-26 16:24:41 +0200123 ~FakeVideoRenderer() override = default;
124
Evan Shrubsoledcb9c5d2022-06-13 15:39:53 +0000125 void OnFrame(const VideoFrame& frame) override {
126 RTC_LOG(LS_VERBOSE) << "Received frame with timestamp="
127 << frame.timestamp();
Evan Shrubsole7cbd8de2022-08-16 08:08:53 +0000128 if (!last_frame_.empty()) {
129 RTC_LOG(LS_INFO) << "Already had frame queue with timestamp="
130 << last_frame_.back().timestamp();
Evan Shrubsoledcb9c5d2022-06-13 15:39:53 +0000131 }
Evan Shrubsole7cbd8de2022-08-16 08:08:53 +0000132 last_frame_.push_back(frame);
Evan Shrubsole44be5792022-04-26 16:24:41 +0200133 }
134
Evan Shrubsoledcb9c5d2022-06-13 15:39:53 +0000135 // If `advance_time`, then the clock will always advance by `timeout`.
136 absl::optional<VideoFrame> WaitForFrame(TimeDelta timeout,
137 bool advance_time = false) {
138 auto start = time_controller_->GetClock()->CurrentTime();
Evan Shrubsole7cbd8de2022-08-16 08:08:53 +0000139 if (last_frame_.empty()) {
Evan Shrubsolea4062722022-05-02 17:46:07 +0200140 time_controller_->AdvanceTime(TimeDelta::Zero());
Evan Shrubsole7cbd8de2022-08-16 08:08:53 +0000141 time_controller_->Wait([this] { return !last_frame_.empty(); }, timeout);
Evan Shrubsole44be5792022-04-26 16:24:41 +0200142 }
Evan Shrubsole7cbd8de2022-08-16 08:08:53 +0000143 absl::optional<VideoFrame> ret;
144 if (!last_frame_.empty()) {
145 ret = last_frame_.front();
146 last_frame_.pop_front();
147 }
Evan Shrubsoledcb9c5d2022-06-13 15:39:53 +0000148 if (advance_time) {
149 time_controller_->AdvanceTime(
150 timeout - (time_controller_->GetClock()->CurrentTime() - start));
151 }
Evan Shrubsole44be5792022-04-26 16:24:41 +0200152 return ret;
153 }
154
155 private:
Evan Shrubsole7cbd8de2022-08-16 08:08:53 +0000156 std::deque<VideoFrame> last_frame_;
Evan Shrubsolea4062722022-05-02 17:46:07 +0200157 TimeController* const time_controller_;
Evan Shrubsole44be5792022-04-26 16:24:41 +0200158};
159
Jonas Oreland0deda152022-09-23 12:08:57 +0200160MATCHER_P2(MatchResolution, w, h, "") {
Evan Shrubsole44be5792022-04-26 16:24:41 +0200161 return arg.resolution().width == w && arg.resolution().height == h;
162}
163
Evan Shrubsoledcb9c5d2022-06-13 15:39:53 +0000164MATCHER_P(RtpTimestamp, timestamp, "") {
Evan Shrubsole7cbd8de2022-08-16 08:08:53 +0000165 if (arg.timestamp() != timestamp) {
166 *result_listener->stream()
167 << "rtp timestamp was " << arg.timestamp() << " != " << timestamp;
168 return false;
169 }
170 return true;
Evan Shrubsolea4062722022-05-02 17:46:07 +0200171}
172
173// Rtp timestamp for in order frame at 30fps.
174uint32_t RtpTimestampForFrame(int id) {
175 return kFirstRtpTimestamp + id * k30FpsRtpTimestampDelta;
176}
177
178// Receive time for in order frame at 30fps.
179Timestamp ReceiveTimeForFrame(int id) {
180 return kStartTime + id * k30FpsDelay;
181}
182
Tommiae4d0972020-05-18 08:45:38 +0200183} // namespace
184
Evan Shrubsolea4062722022-05-02 17:46:07 +0200185class VideoReceiveStream2Test : public ::testing::TestWithParam<bool> {
Tommiae4d0972020-05-18 08:45:38 +0200186 public:
Evan Shrubsolea4062722022-05-02 17:46:07 +0200187 auto DefaultDecodeAction() {
philipel27b35a72022-07-05 09:59:55 +0200188 return Invoke(&fake_decoder_, &test::FakeDecoder::Decode);
Evan Shrubsolea4062722022-05-02 17:46:07 +0200189 }
190
191 bool UseMetronome() const { return GetParam(); }
192
Tommiae4d0972020-05-18 08:45:38 +0200193 VideoReceiveStream2Test()
Evan Shrubsole44be5792022-04-26 16:24:41 +0200194 : time_controller_(kStartTime),
195 clock_(time_controller_.GetClock()),
Evan Shrubsole1c184772022-04-26 09:47:49 +0200196 config_(&mock_transport_, &mock_h264_decoder_factory_),
Evan Shrubsole7cbd8de2022-08-16 08:08:53 +0000197 call_stats_(clock_, time_controller_.GetMainThread()),
198 fake_renderer_(&time_controller_),
Markus Handellbe400e42022-11-08 12:14:23 +0100199 fake_metronome_(TimeDelta::Millis(16)),
Evan Shrubsole7cbd8de2022-08-16 08:08:53 +0000200 decode_sync_(clock_,
201 &fake_metronome_,
202 time_controller_.GetMainThread()),
Evan Shrubsolea4062722022-05-02 17:46:07 +0200203 h264_decoder_factory_(&mock_decoder_) {
Evan Shrubsole1c184772022-04-26 09:47:49 +0200204 // By default, mock decoder factory is backed by VideoDecoderProxyFactory.
Evan Shrubsole14ee8032022-04-26 16:01:44 +0200205 ON_CALL(mock_h264_decoder_factory_, CreateVideoDecoder)
philipel27b35a72022-07-05 09:59:55 +0200206 .WillByDefault(
207 Invoke(&h264_decoder_factory_,
208 &test::VideoDecoderProxyFactory::CreateVideoDecoder));
Evan Shrubsole14ee8032022-04-26 16:01:44 +0200209
210 // By default, mock decode will wrap the fake decoder.
Evan Shrubsolea4062722022-05-02 17:46:07 +0200211 ON_CALL(mock_decoder_, Configure)
philipel27b35a72022-07-05 09:59:55 +0200212 .WillByDefault(Invoke(&fake_decoder_, &test::FakeDecoder::Configure));
Evan Shrubsolea4062722022-05-02 17:46:07 +0200213 ON_CALL(mock_decoder_, Decode).WillByDefault(DefaultDecodeAction());
214 ON_CALL(mock_decoder_, RegisterDecodeCompleteCallback)
Evan Shrubsole14ee8032022-04-26 16:01:44 +0200215 .WillByDefault(
philipel27b35a72022-07-05 09:59:55 +0200216 Invoke(&fake_decoder_,
217 &test::FakeDecoder::RegisterDecodeCompleteCallback));
218 ON_CALL(mock_decoder_, Release)
219 .WillByDefault(Invoke(&fake_decoder_, &test::FakeDecoder::Release));
220 ON_CALL(mock_transport_, SendRtcp)
221 .WillByDefault(
222 Invoke(&rtcp_packet_parser_, &test::RtcpPacketParser::Parse));
Jonas Oreland8ca06132022-03-14 12:52:48 +0100223 }
Evan Shrubsole7cbd8de2022-08-16 08:08:53 +0000224
Tommi90738dd2021-05-31 17:36:47 +0200225 ~VideoReceiveStream2Test() override {
Evan Shrubsolea4062722022-05-02 17:46:07 +0200226 if (video_receive_stream_) {
227 video_receive_stream_->Stop();
Tommi90738dd2021-05-31 17:36:47 +0200228 video_receive_stream_->UnregisterFromTransport();
Evan Shrubsolea4062722022-05-02 17:46:07 +0200229 }
Evan Shrubsolea4062722022-05-02 17:46:07 +0200230 time_controller_.AdvanceTime(TimeDelta::Zero());
Tommi90738dd2021-05-31 17:36:47 +0200231 }
Tommiae4d0972020-05-18 08:45:38 +0200232
Tommi90738dd2021-05-31 17:36:47 +0200233 void SetUp() override {
Tommiae4d0972020-05-18 08:45:38 +0200234 config_.rtp.remote_ssrc = 1111;
235 config_.rtp.local_ssrc = 2222;
236 config_.renderer = &fake_renderer_;
Tommif6f45432022-05-20 15:21:20 +0200237 VideoReceiveStreamInterface::Decoder h264_decoder;
Tommiae4d0972020-05-18 08:45:38 +0200238 h264_decoder.payload_type = 99;
239 h264_decoder.video_format = SdpVideoFormat("H264");
240 h264_decoder.video_format.parameters.insert(
241 {"sprop-parameter-sets", "Z0IACpZTBYmI,aMljiA=="});
Tommif6f45432022-05-20 15:21:20 +0200242 VideoReceiveStreamInterface::Decoder h265_decoder;
Evan Shrubsole1c184772022-04-26 09:47:49 +0200243 h265_decoder.payload_type = 100;
244 h265_decoder.video_format = SdpVideoFormat("H265");
245
246 config_.decoders = {h265_decoder, h264_decoder};
Tommiae4d0972020-05-18 08:45:38 +0200247
Evan Shrubsole44be5792022-04-26 16:24:41 +0200248 RecreateReceiveStream();
Evan Shrubsole14ee8032022-04-26 16:01:44 +0200249 }
250
Tommif6f45432022-05-20 15:21:20 +0200251 void RecreateReceiveStream(
252 absl::optional<VideoReceiveStreamInterface::RecordingState> state =
253 absl::nullopt) {
Evan Shrubsole14ee8032022-04-26 16:01:44 +0200254 if (video_receive_stream_) {
255 video_receive_stream_->UnregisterFromTransport();
256 video_receive_stream_ = nullptr;
257 }
258 timing_ = new VCMTiming(clock_, fake_call_.trials());
Tommiae4d0972020-05-18 08:45:38 +0200259 video_receive_stream_ =
260 std::make_unique<webrtc::internal::VideoReceiveStream2>(
Evan Shrubsole44be5792022-04-26 16:24:41 +0200261 time_controller_.GetTaskQueueFactory(), &fake_call_,
262 kDefaultNumCpuCores, &packet_router_, config_.Copy(), &call_stats_,
263 clock_, absl::WrapUnique(timing_), &nack_periodic_processor_,
Evan Shrubsolea006ba12022-09-05 10:09:08 +0000264 UseMetronome() ? &decode_sync_ : nullptr, nullptr);
Tommi90738dd2021-05-31 17:36:47 +0200265 video_receive_stream_->RegisterWithTransport(
266 &rtp_stream_receiver_controller_);
Evan Shrubsole44be5792022-04-26 16:24:41 +0200267 if (state)
268 video_receive_stream_->SetAndGetRecordingState(std::move(*state), false);
Tommiae4d0972020-05-18 08:45:38 +0200269 }
270
271 protected:
Evan Shrubsole44be5792022-04-26 16:24:41 +0200272 GlobalSimulatedTimeController time_controller_;
273 Clock* const clock_;
Markus Handell0e62f7a2021-07-20 13:32:02 +0200274 NackPeriodicProcessor nack_periodic_processor_;
Evan Shrubsole1c184772022-04-26 09:47:49 +0200275 testing::NiceMock<MockVideoDecoderFactory> mock_h264_decoder_factory_;
Tommif6f45432022-05-20 15:21:20 +0200276 VideoReceiveStreamInterface::Config config_;
Tommiae4d0972020-05-18 08:45:38 +0200277 internal::CallStats call_stats_;
Evan Shrubsolea4062722022-05-02 17:46:07 +0200278 testing::NiceMock<MockVideoDecoder> mock_decoder_;
Evan Shrubsole44be5792022-04-26 16:24:41 +0200279 FakeVideoRenderer fake_renderer_;
Tommi90738dd2021-05-31 17:36:47 +0200280 cricket::FakeCall fake_call_;
Tommiae4d0972020-05-18 08:45:38 +0200281 MockTransport mock_transport_;
philipel27b35a72022-07-05 09:59:55 +0200282 test::RtcpPacketParser rtcp_packet_parser_;
Tommiae4d0972020-05-18 08:45:38 +0200283 PacketRouter packet_router_;
284 RtpStreamReceiverController rtp_stream_receiver_controller_;
285 std::unique_ptr<webrtc::internal::VideoReceiveStream2> video_receive_stream_;
Tommiae4d0972020-05-18 08:45:38 +0200286 VCMTiming* timing_;
Evan Shrubsolea4062722022-05-02 17:46:07 +0200287 test::FakeMetronome fake_metronome_;
288 DecodeSynchronizer decode_sync_;
Evan Shrubsole1c184772022-04-26 09:47:49 +0200289
290 private:
291 test::VideoDecoderProxyFactory h264_decoder_factory_;
Evan Shrubsole14ee8032022-04-26 16:01:44 +0200292 test::FakeDecoder fake_decoder_;
Tommiae4d0972020-05-18 08:45:38 +0200293};
294
Evan Shrubsolea4062722022-05-02 17:46:07 +0200295TEST_P(VideoReceiveStream2Test, CreateFrameFromH264FmtpSpropAndIdr) {
Tommiae4d0972020-05-18 08:45:38 +0200296 constexpr uint8_t idr_nalu[] = {0x05, 0xFF, 0xFF, 0xFF};
297 RtpPacketToSend rtppacket(nullptr);
298 uint8_t* payload = rtppacket.AllocatePayload(sizeof(idr_nalu));
299 memcpy(payload, idr_nalu, sizeof(idr_nalu));
300 rtppacket.SetMarker(true);
301 rtppacket.SetSsrc(1111);
302 rtppacket.SetPayloadType(99);
303 rtppacket.SetSequenceNumber(1);
304 rtppacket.SetTimestamp(0);
Evan Shrubsolea4062722022-05-02 17:46:07 +0200305 EXPECT_CALL(mock_decoder_, RegisterDecodeCompleteCallback(_));
Tommiae4d0972020-05-18 08:45:38 +0200306 video_receive_stream_->Start();
Evan Shrubsolea4062722022-05-02 17:46:07 +0200307 EXPECT_CALL(mock_decoder_, Decode(_, false, _));
Tommiae4d0972020-05-18 08:45:38 +0200308 RtpPacketReceived parsed_packet;
309 ASSERT_TRUE(parsed_packet.Parse(rtppacket.data(), rtppacket.size()));
310 rtp_stream_receiver_controller_.OnRtpPacket(parsed_packet);
Evan Shrubsolea4062722022-05-02 17:46:07 +0200311 EXPECT_CALL(mock_decoder_, Release());
Evan Shrubsole44be5792022-04-26 16:24:41 +0200312
313 time_controller_.AdvanceTime(TimeDelta::Zero());
Tommiae4d0972020-05-18 08:45:38 +0200314}
315
Evan Shrubsolea4062722022-05-02 17:46:07 +0200316TEST_P(VideoReceiveStream2Test, PlayoutDelay) {
Niels Möllerd381eed2020-09-02 15:34:40 +0200317 const VideoPlayoutDelay kPlayoutDelayMs = {123, 321};
Evan Shrubsolea0ee64c2022-04-26 10:09:04 +0200318 std::unique_ptr<test::FakeEncodedFrame> test_frame =
319 test::FakeFrameBuilder().Id(0).AsLast().Build();
Tommiae4d0972020-05-18 08:45:38 +0200320 test_frame->SetPlayoutDelay(kPlayoutDelayMs);
321
322 video_receive_stream_->OnCompleteFrame(std::move(test_frame));
Evan Shrubsole8f1159b2022-03-22 12:12:17 +0100323 auto timings = timing_->GetTimings();
324 EXPECT_EQ(kPlayoutDelayMs.min_ms, timings.min_playout_delay.ms());
325 EXPECT_EQ(kPlayoutDelayMs.max_ms, timings.max_playout_delay.ms());
Tommiae4d0972020-05-18 08:45:38 +0200326
327 // Check that the biggest minimum delay is chosen.
328 video_receive_stream_->SetMinimumPlayoutDelay(400);
Evan Shrubsole8f1159b2022-03-22 12:12:17 +0100329 timings = timing_->GetTimings();
330 EXPECT_EQ(400, timings.min_playout_delay.ms());
Tommiae4d0972020-05-18 08:45:38 +0200331
332 // Check base minimum delay validation.
333 EXPECT_FALSE(video_receive_stream_->SetBaseMinimumPlayoutDelayMs(12345));
334 EXPECT_FALSE(video_receive_stream_->SetBaseMinimumPlayoutDelayMs(-1));
335 EXPECT_TRUE(video_receive_stream_->SetBaseMinimumPlayoutDelayMs(500));
Evan Shrubsole8f1159b2022-03-22 12:12:17 +0100336 timings = timing_->GetTimings();
337 EXPECT_EQ(500, timings.min_playout_delay.ms());
Tommiae4d0972020-05-18 08:45:38 +0200338
339 // Check that intermidiate values are remembered and the biggest remembered
340 // is chosen.
341 video_receive_stream_->SetBaseMinimumPlayoutDelayMs(0);
Evan Shrubsole8f1159b2022-03-22 12:12:17 +0100342 timings = timing_->GetTimings();
343 EXPECT_EQ(400, timings.min_playout_delay.ms());
Tommiae4d0972020-05-18 08:45:38 +0200344
345 video_receive_stream_->SetMinimumPlayoutDelay(0);
Evan Shrubsole8f1159b2022-03-22 12:12:17 +0100346 timings = timing_->GetTimings();
347 EXPECT_EQ(123, timings.min_playout_delay.ms());
Tommiae4d0972020-05-18 08:45:38 +0200348}
349
Evan Shrubsolea4062722022-05-02 17:46:07 +0200350TEST_P(VideoReceiveStream2Test, PlayoutDelayPreservesDefaultMaxValue) {
Evan Shrubsole8f1159b2022-03-22 12:12:17 +0100351 const TimeDelta default_max_playout_latency =
352 timing_->GetTimings().max_playout_delay;
Niels Möllerd381eed2020-09-02 15:34:40 +0200353 const VideoPlayoutDelay kPlayoutDelayMs = {123, -1};
Tommiae4d0972020-05-18 08:45:38 +0200354
Evan Shrubsolea0ee64c2022-04-26 10:09:04 +0200355 std::unique_ptr<test::FakeEncodedFrame> test_frame =
356 test::FakeFrameBuilder().Id(0).AsLast().Build();
Tommiae4d0972020-05-18 08:45:38 +0200357 test_frame->SetPlayoutDelay(kPlayoutDelayMs);
358
359 video_receive_stream_->OnCompleteFrame(std::move(test_frame));
360
Artem Titovab30d722021-07-27 16:22:11 +0200361 // Ensure that -1 preserves default maximum value from `timing_`.
Evan Shrubsole8f1159b2022-03-22 12:12:17 +0100362 auto timings = timing_->GetTimings();
363 EXPECT_EQ(kPlayoutDelayMs.min_ms, timings.min_playout_delay.ms());
364 EXPECT_NE(kPlayoutDelayMs.max_ms, timings.max_playout_delay.ms());
365 EXPECT_EQ(default_max_playout_latency, timings.max_playout_delay);
Tommiae4d0972020-05-18 08:45:38 +0200366}
367
Evan Shrubsolea4062722022-05-02 17:46:07 +0200368TEST_P(VideoReceiveStream2Test, PlayoutDelayPreservesDefaultMinValue) {
Evan Shrubsole8f1159b2022-03-22 12:12:17 +0100369 const TimeDelta default_min_playout_latency =
370 timing_->GetTimings().min_playout_delay;
Niels Möllerd381eed2020-09-02 15:34:40 +0200371 const VideoPlayoutDelay kPlayoutDelayMs = {-1, 321};
Tommiae4d0972020-05-18 08:45:38 +0200372
Evan Shrubsolea0ee64c2022-04-26 10:09:04 +0200373 std::unique_ptr<test::FakeEncodedFrame> test_frame =
374 test::FakeFrameBuilder().Id(0).AsLast().Build();
Tommiae4d0972020-05-18 08:45:38 +0200375 test_frame->SetPlayoutDelay(kPlayoutDelayMs);
376
377 video_receive_stream_->OnCompleteFrame(std::move(test_frame));
378
Artem Titovab30d722021-07-27 16:22:11 +0200379 // Ensure that -1 preserves default minimum value from `timing_`.
Evan Shrubsole8f1159b2022-03-22 12:12:17 +0100380 auto timings = timing_->GetTimings();
381 EXPECT_NE(kPlayoutDelayMs.min_ms, timings.min_playout_delay.ms());
382 EXPECT_EQ(kPlayoutDelayMs.max_ms, timings.max_playout_delay.ms());
383 EXPECT_EQ(default_min_playout_latency, timings.min_playout_delay);
Tommiae4d0972020-05-18 08:45:38 +0200384}
385
Johannes Kronbbf639e2022-06-15 12:27:23 +0200386TEST_P(VideoReceiveStream2Test, RenderParametersSetToDefaultValues) {
387 // Default render parameters.
388 const VideoFrame::RenderParameters kDefaultRenderParameters;
Johannes Kron111e9812020-10-26 13:54:40 +0100389 // Default with no playout delay set.
Evan Shrubsolea0ee64c2022-04-26 10:09:04 +0200390 std::unique_ptr<test::FakeEncodedFrame> test_frame0 =
391 test::FakeFrameBuilder().Id(0).AsLast().Build();
Johannes Kron111e9812020-10-26 13:54:40 +0100392 video_receive_stream_->OnCompleteFrame(std::move(test_frame0));
Johannes Kronbbf639e2022-06-15 12:27:23 +0200393 EXPECT_EQ(timing_->RenderParameters(), kDefaultRenderParameters);
394}
Johannes Kron111e9812020-10-26 13:54:40 +0100395
Johannes Kronbbf639e2022-06-15 12:27:23 +0200396TEST_P(VideoReceiveStream2Test, UseLowLatencyRenderingSetFromPlayoutDelay) {
397 // use_low_latency_rendering set if playout delay set to min=0, max<=500 ms.
398 std::unique_ptr<test::FakeEncodedFrame> test_frame0 =
399 test::FakeFrameBuilder().Id(0).AsLast().Build();
400 test_frame0->SetPlayoutDelay({/*min_ms=*/0, /*max_ms=*/0});
401 video_receive_stream_->OnCompleteFrame(std::move(test_frame0));
402 EXPECT_TRUE(timing_->RenderParameters().use_low_latency_rendering);
403
Evan Shrubsolea0ee64c2022-04-26 10:09:04 +0200404 std::unique_ptr<test::FakeEncodedFrame> test_frame1 =
405 test::FakeFrameBuilder().Id(1).AsLast().Build();
Johannes Kronbbf639e2022-06-15 12:27:23 +0200406 test_frame1->SetPlayoutDelay({/*min_ms=*/0, /*max_ms=*/500});
Johannes Kron111e9812020-10-26 13:54:40 +0100407 video_receive_stream_->OnCompleteFrame(std::move(test_frame1));
Johannes Kronbbf639e2022-06-15 12:27:23 +0200408 EXPECT_TRUE(timing_->RenderParameters().use_low_latency_rendering);
Johannes Kron111e9812020-10-26 13:54:40 +0100409}
410
Evan Shrubsolea4062722022-05-02 17:46:07 +0200411TEST_P(VideoReceiveStream2Test, MaxCompositionDelaySetFromMaxPlayoutDelay) {
Johannes Kronbbf639e2022-06-15 12:27:23 +0200412 // The max composition delay is dependent on the number of frames in the
413 // pre-decode queue. It's therefore important to advance the time as the test
414 // runs to get the correct expectations of max_composition_delay_in_frames.
415 video_receive_stream_->Start();
416 // Max composition delay not set if no playout delay is set.
417 std::unique_ptr<test::FakeEncodedFrame> test_frame0 =
418 test::FakeFrameBuilder()
419 .Id(0)
420 .Time(RtpTimestampForFrame(0))
421 .ReceivedTime(ReceiveTimeForFrame(0))
422 .AsLast()
423 .Build();
424 video_receive_stream_->OnCompleteFrame(std::move(test_frame0));
425 EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames,
426 Eq(absl::nullopt));
427 time_controller_.AdvanceTime(k30FpsDelay);
Johannes Kronbbf639e2022-06-15 12:27:23 +0200428
429 // Max composition delay not set for playout delay 0,0.
430 std::unique_ptr<test::FakeEncodedFrame> test_frame1 =
431 test::FakeFrameBuilder()
432 .Id(1)
433 .Time(RtpTimestampForFrame(1))
434 .ReceivedTime(ReceiveTimeForFrame(1))
435 .AsLast()
436 .Build();
437 test_frame1->SetPlayoutDelay({0, 0});
438 video_receive_stream_->OnCompleteFrame(std::move(test_frame1));
439 EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames,
440 Eq(absl::nullopt));
441 time_controller_.AdvanceTime(k30FpsDelay);
Johannes Kronbbf639e2022-06-15 12:27:23 +0200442
443 // Max composition delay not set for playout delay X,Y, where X,Y>0.
444 std::unique_ptr<test::FakeEncodedFrame> test_frame2 =
445 test::FakeFrameBuilder()
446 .Id(2)
447 .Time(RtpTimestampForFrame(2))
448 .ReceivedTime(ReceiveTimeForFrame(2))
449 .AsLast()
450 .Build();
451 test_frame2->SetPlayoutDelay({10, 30});
452 video_receive_stream_->OnCompleteFrame(std::move(test_frame2));
453 EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames,
454 Eq(absl::nullopt));
455
456 time_controller_.AdvanceTime(k30FpsDelay);
Johannes Kronbbf639e2022-06-15 12:27:23 +0200457
Johannes Kron111e9812020-10-26 13:54:40 +0100458 // Max composition delay set if playout delay X,Y, where X=0,Y>0.
Johannes Kron111e9812020-10-26 13:54:40 +0100459 const int kExpectedMaxCompositionDelayInFrames = 3; // ~50 ms at 60 fps.
Johannes Kronbbf639e2022-06-15 12:27:23 +0200460 std::unique_ptr<test::FakeEncodedFrame> test_frame3 =
461 test::FakeFrameBuilder()
462 .Id(3)
463 .Time(RtpTimestampForFrame(3))
464 .ReceivedTime(ReceiveTimeForFrame(3))
465 .AsLast()
466 .Build();
467 test_frame3->SetPlayoutDelay({0, 50});
468 video_receive_stream_->OnCompleteFrame(std::move(test_frame3));
469 EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames,
470 Optional(kExpectedMaxCompositionDelayInFrames));
Johannes Kron111e9812020-10-26 13:54:40 +0100471}
472
Evan Shrubsolea4062722022-05-02 17:46:07 +0200473TEST_P(VideoReceiveStream2Test, LazyDecoderCreation) {
Evan Shrubsole1c184772022-04-26 09:47:49 +0200474 constexpr uint8_t idr_nalu[] = {0x05, 0xFF, 0xFF, 0xFF};
475 RtpPacketToSend rtppacket(nullptr);
476 uint8_t* payload = rtppacket.AllocatePayload(sizeof(idr_nalu));
477 memcpy(payload, idr_nalu, sizeof(idr_nalu));
478 rtppacket.SetMarker(true);
479 rtppacket.SetSsrc(1111);
480 // H265 payload type.
481 rtppacket.SetPayloadType(99);
482 rtppacket.SetSequenceNumber(1);
483 rtppacket.SetTimestamp(0);
484
Johannes Kronbb591c42022-09-28 10:10:25 +0000485 // No decoders are created by default.
Evan Shrubsole1c184772022-04-26 09:47:49 +0200486 EXPECT_CALL(mock_h264_decoder_factory_, CreateVideoDecoder(_)).Times(0);
Evan Shrubsole1c184772022-04-26 09:47:49 +0200487 video_receive_stream_->Start();
Erik Språng7aaeb5a2022-08-18 13:18:27 +0200488 time_controller_.AdvanceTime(TimeDelta::Zero());
Evan Shrubsole1c184772022-04-26 09:47:49 +0200489
490 EXPECT_TRUE(
491 testing::Mock::VerifyAndClearExpectations(&mock_h264_decoder_factory_));
Johannes Kronbb591c42022-09-28 10:10:25 +0000492 // Verify that the decoder is created when we receive payload data and tries
493 // to decode a frame.
Erik Språng7aaeb5a2022-08-18 13:18:27 +0200494 EXPECT_CALL(
495 mock_h264_decoder_factory_,
496 CreateVideoDecoder(Field(&SdpVideoFormat::name, testing::Eq("H264"))));
497 EXPECT_CALL(mock_decoder_, Configure);
498 EXPECT_CALL(mock_decoder_, RegisterDecodeCompleteCallback);
499 EXPECT_CALL(mock_decoder_, Decode);
Evan Shrubsole1c184772022-04-26 09:47:49 +0200500 RtpPacketReceived parsed_packet;
501 ASSERT_TRUE(parsed_packet.Parse(rtppacket.data(), rtppacket.size()));
502 rtp_stream_receiver_controller_.OnRtpPacket(parsed_packet);
Erik Språng7aaeb5a2022-08-18 13:18:27 +0200503 EXPECT_CALL(mock_decoder_, Release);
Evan Shrubsole1c184772022-04-26 09:47:49 +0200504
505 // Make sure the decoder thread had a chance to run.
Erik Språng7aaeb5a2022-08-18 13:18:27 +0200506 time_controller_.AdvanceTime(TimeDelta::Zero());
Evan Shrubsole1c184772022-04-26 09:47:49 +0200507}
508
Evan Shrubsolea4062722022-05-02 17:46:07 +0200509TEST_P(VideoReceiveStream2Test, PassesNtpTime) {
Evan Shrubsolea0ee64c2022-04-26 10:09:04 +0200510 const Timestamp kNtpTimestamp = Timestamp::Millis(12345);
511 std::unique_ptr<test::FakeEncodedFrame> test_frame =
512 test::FakeFrameBuilder()
513 .Id(0)
514 .PayloadType(99)
515 .NtpTime(kNtpTimestamp)
516 .AsLast()
517 .Build();
Tommiae4d0972020-05-18 08:45:38 +0200518
519 video_receive_stream_->Start();
520 video_receive_stream_->OnCompleteFrame(std::move(test_frame));
Evan Shrubsolea4062722022-05-02 17:46:07 +0200521 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut),
Evan Shrubsole44be5792022-04-26 16:24:41 +0200522 RenderedFrameWith(NtpTimestamp(kNtpTimestamp)));
Tommiae4d0972020-05-18 08:45:38 +0200523}
524
Evan Shrubsolea4062722022-05-02 17:46:07 +0200525TEST_P(VideoReceiveStream2Test, PassesRotation) {
Tommiae4d0972020-05-18 08:45:38 +0200526 const webrtc::VideoRotation kRotation = webrtc::kVideoRotation_180;
Evan Shrubsolea0ee64c2022-04-26 10:09:04 +0200527 std::unique_ptr<test::FakeEncodedFrame> test_frame = test::FakeFrameBuilder()
528 .Id(0)
529 .PayloadType(99)
530 .Rotation(kRotation)
531 .AsLast()
532 .Build();
Tommiae4d0972020-05-18 08:45:38 +0200533
534 video_receive_stream_->Start();
535 video_receive_stream_->OnCompleteFrame(std::move(test_frame));
Evan Shrubsolea4062722022-05-02 17:46:07 +0200536 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut),
Evan Shrubsole44be5792022-04-26 16:24:41 +0200537 RenderedFrameWith(Rotation(kRotation)));
Tommiae4d0972020-05-18 08:45:38 +0200538}
539
Evan Shrubsolea4062722022-05-02 17:46:07 +0200540TEST_P(VideoReceiveStream2Test, PassesPacketInfos) {
Tommiae4d0972020-05-18 08:45:38 +0200541 RtpPacketInfos packet_infos = CreatePacketInfos(3);
Evan Shrubsolea0ee64c2022-04-26 10:09:04 +0200542 auto test_frame = test::FakeFrameBuilder()
543 .Id(0)
544 .PayloadType(99)
545 .PacketInfos(packet_infos)
546 .AsLast()
547 .Build();
Tommiae4d0972020-05-18 08:45:38 +0200548
549 video_receive_stream_->Start();
550 video_receive_stream_->OnCompleteFrame(std::move(test_frame));
Evan Shrubsolea4062722022-05-02 17:46:07 +0200551 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut),
Evan Shrubsole44be5792022-04-26 16:24:41 +0200552 RenderedFrameWith(PacketInfos(ElementsAreArray(packet_infos))));
Tommiae4d0972020-05-18 08:45:38 +0200553}
554
Evan Shrubsolea4062722022-05-02 17:46:07 +0200555TEST_P(VideoReceiveStream2Test, RenderedFrameUpdatesGetSources) {
Tommiae4d0972020-05-18 08:45:38 +0200556 constexpr uint32_t kSsrc = 1111;
557 constexpr uint32_t kCsrc = 9001;
558 constexpr uint32_t kRtpTimestamp = 12345;
559
560 // Prepare one video frame with per-packet information.
Evan Shrubsolea0ee64c2022-04-26 10:09:04 +0200561 auto test_frame =
562 test::FakeFrameBuilder().Id(0).PayloadType(99).AsLast().Build();
Tommiae4d0972020-05-18 08:45:38 +0200563 RtpPacketInfos packet_infos;
564 {
565 RtpPacketInfos::vector_type infos;
566
567 RtpPacketInfo info;
568 info.set_ssrc(kSsrc);
569 info.set_csrcs({kCsrc});
570 info.set_rtp_timestamp(kRtpTimestamp);
571
Johannes Kronf7de74c2021-04-30 13:10:56 +0200572 info.set_receive_time(clock_->CurrentTime() - TimeDelta::Millis(5000));
Tommiae4d0972020-05-18 08:45:38 +0200573 infos.push_back(info);
574
Johannes Kronf7de74c2021-04-30 13:10:56 +0200575 info.set_receive_time(clock_->CurrentTime() - TimeDelta::Millis(3000));
Tommiae4d0972020-05-18 08:45:38 +0200576 infos.push_back(info);
577
Johannes Kronf7de74c2021-04-30 13:10:56 +0200578 info.set_receive_time(clock_->CurrentTime() - TimeDelta::Millis(2000));
Tommiae4d0972020-05-18 08:45:38 +0200579 infos.push_back(info);
580
Johannes Kronf7de74c2021-04-30 13:10:56 +0200581 info.set_receive_time(clock_->CurrentTime() - TimeDelta::Millis(1000));
Tommiae4d0972020-05-18 08:45:38 +0200582 infos.push_back(info);
583
584 packet_infos = RtpPacketInfos(std::move(infos));
585 }
586 test_frame->SetPacketInfos(packet_infos);
587
588 // Start receive stream.
589 video_receive_stream_->Start();
590 EXPECT_THAT(video_receive_stream_->GetSources(), IsEmpty());
591
592 // Render one video frame.
593 int64_t timestamp_ms_min = clock_->TimeInMilliseconds();
594 video_receive_stream_->OnCompleteFrame(std::move(test_frame));
Tommiae4d0972020-05-18 08:45:38 +0200595 // Verify that the per-packet information is passed to the renderer.
Evan Shrubsolea4062722022-05-02 17:46:07 +0200596 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut),
Evan Shrubsole44be5792022-04-26 16:24:41 +0200597 RenderedFrameWith(PacketInfos(ElementsAreArray(packet_infos))));
598 int64_t timestamp_ms_max = clock_->TimeInMilliseconds();
Tommiae4d0972020-05-18 08:45:38 +0200599
Artem Titovab30d722021-07-27 16:22:11 +0200600 // Verify that the per-packet information also updates `GetSources()`.
Tommiae4d0972020-05-18 08:45:38 +0200601 std::vector<RtpSource> sources = video_receive_stream_->GetSources();
602 ASSERT_THAT(sources, SizeIs(2));
603 {
604 auto it = std::find_if(sources.begin(), sources.end(),
605 [](const RtpSource& source) {
606 return source.source_type() == RtpSourceType::SSRC;
607 });
608 ASSERT_NE(it, sources.end());
609
610 EXPECT_EQ(it->source_id(), kSsrc);
611 EXPECT_EQ(it->source_type(), RtpSourceType::SSRC);
612 EXPECT_EQ(it->rtp_timestamp(), kRtpTimestamp);
613 EXPECT_GE(it->timestamp_ms(), timestamp_ms_min);
614 EXPECT_LE(it->timestamp_ms(), timestamp_ms_max);
615 }
616 {
617 auto it = std::find_if(sources.begin(), sources.end(),
618 [](const RtpSource& source) {
619 return source.source_type() == RtpSourceType::CSRC;
620 });
621 ASSERT_NE(it, sources.end());
622
623 EXPECT_EQ(it->source_id(), kCsrc);
624 EXPECT_EQ(it->source_type(), RtpSourceType::CSRC);
625 EXPECT_EQ(it->rtp_timestamp(), kRtpTimestamp);
626 EXPECT_GE(it->timestamp_ms(), timestamp_ms_min);
627 EXPECT_LE(it->timestamp_ms(), timestamp_ms_max);
628 }
629}
630
Evan Shrubsolea0ee64c2022-04-26 10:09:04 +0200631std::unique_ptr<test::FakeEncodedFrame> MakeFrameWithResolution(
Markus Handell588f9b32021-04-08 19:19:50 +0200632 VideoFrameType frame_type,
633 int picture_id,
634 int width,
635 int height) {
Evan Shrubsolea0ee64c2022-04-26 10:09:04 +0200636 auto frame =
637 test::FakeFrameBuilder().Id(picture_id).PayloadType(99).AsLast().Build();
Tommiae4d0972020-05-18 08:45:38 +0200638 frame->SetFrameType(frame_type);
Markus Handell588f9b32021-04-08 19:19:50 +0200639 frame->_encodedWidth = width;
640 frame->_encodedHeight = height;
Tommiae4d0972020-05-18 08:45:38 +0200641 return frame;
642}
643
Evan Shrubsolea0ee64c2022-04-26 10:09:04 +0200644std::unique_ptr<test::FakeEncodedFrame> MakeFrame(VideoFrameType frame_type,
645 int picture_id) {
Markus Handell588f9b32021-04-08 19:19:50 +0200646 return MakeFrameWithResolution(frame_type, picture_id, 320, 240);
647}
648
Evan Shrubsolea4062722022-05-02 17:46:07 +0200649TEST_P(VideoReceiveStream2Test, PassesFrameWhenEncodedFramesCallbackSet) {
Tommiae4d0972020-05-18 08:45:38 +0200650 testing::MockFunction<void(const RecordableEncodedFrame&)> callback;
651 video_receive_stream_->Start();
Tommiae4d0972020-05-18 08:45:38 +0200652 EXPECT_CALL(callback, Call);
653 video_receive_stream_->SetAndGetRecordingState(
Tommif6f45432022-05-20 15:21:20 +0200654 VideoReceiveStreamInterface::RecordingState(callback.AsStdFunction()),
655 true);
Tommiae4d0972020-05-18 08:45:38 +0200656 video_receive_stream_->OnCompleteFrame(
657 MakeFrame(VideoFrameType::kVideoFrameKey, 0));
Evan Shrubsolea4062722022-05-02 17:46:07 +0200658 EXPECT_TRUE(fake_renderer_.WaitForFrame(kDefaultTimeOut));
philipel27b35a72022-07-05 09:59:55 +0200659
660 EXPECT_THAT(rtcp_packet_parser_.pli()->num_packets(), Eq(1));
661
Tommiae4d0972020-05-18 08:45:38 +0200662 video_receive_stream_->Stop();
663}
664
Evan Shrubsolea4062722022-05-02 17:46:07 +0200665TEST_P(VideoReceiveStream2Test, MovesEncodedFrameDispatchStateWhenReCreating) {
Tommiae4d0972020-05-18 08:45:38 +0200666 testing::MockFunction<void(const RecordableEncodedFrame&)> callback;
667 video_receive_stream_->Start();
668 // Expect a key frame request over RTCP.
Tommiae4d0972020-05-18 08:45:38 +0200669 video_receive_stream_->SetAndGetRecordingState(
Tommif6f45432022-05-20 15:21:20 +0200670 VideoReceiveStreamInterface::RecordingState(callback.AsStdFunction()),
671 true);
Tommiae4d0972020-05-18 08:45:38 +0200672 video_receive_stream_->Stop();
Tommif6f45432022-05-20 15:21:20 +0200673 VideoReceiveStreamInterface::RecordingState old_state =
Tommiae4d0972020-05-18 08:45:38 +0200674 video_receive_stream_->SetAndGetRecordingState(
Tommif6f45432022-05-20 15:21:20 +0200675 VideoReceiveStreamInterface::RecordingState(), false);
Evan Shrubsole44be5792022-04-26 16:24:41 +0200676 RecreateReceiveStream(std::move(old_state));
philipel27b35a72022-07-05 09:59:55 +0200677
678 EXPECT_THAT(rtcp_packet_parser_.pli()->num_packets(), Eq(1));
679
Tommiae4d0972020-05-18 08:45:38 +0200680 video_receive_stream_->Stop();
681}
682
Evan Shrubsolea4062722022-05-02 17:46:07 +0200683TEST_P(VideoReceiveStream2Test, RequestsKeyFramesUntilKeyFrameReceived) {
Evan Shrubsole44be5792022-04-26 16:24:41 +0200684 // Recreate receive stream with shorter delay to test rtx.
685 TimeDelta rtx_delay = TimeDelta::Millis(50);
686 config_.rtp.nack.rtp_history_ms = rtx_delay.ms();
687 auto tick = rtx_delay / 2;
688 RecreateReceiveStream();
689 video_receive_stream_->Start();
Markus Handell588f9b32021-04-08 19:19:50 +0200690
Evan Shrubsole44be5792022-04-26 16:24:41 +0200691 video_receive_stream_->GenerateKeyFrame();
692 video_receive_stream_->OnCompleteFrame(
693 MakeFrame(VideoFrameType::kVideoFrameDelta, 0));
Evan Shrubsolea4062722022-05-02 17:46:07 +0200694 fake_renderer_.WaitForFrame(kDefaultTimeOut);
Tommiae4d0972020-05-18 08:45:38 +0200695 time_controller_.AdvanceTime(tick);
Evan Shrubsole44be5792022-04-26 16:24:41 +0200696 video_receive_stream_->OnCompleteFrame(
697 MakeFrame(VideoFrameType::kVideoFrameDelta, 1));
Evan Shrubsolea4062722022-05-02 17:46:07 +0200698 fake_renderer_.WaitForFrame(kDefaultTimeOut);
Evan Shrubsole44be5792022-04-26 16:24:41 +0200699 time_controller_.AdvanceTime(TimeDelta::Zero());
Tommiae4d0972020-05-18 08:45:38 +0200700 testing::Mock::VerifyAndClearExpectations(&mock_transport_);
701
philipel27b35a72022-07-05 09:59:55 +0200702 EXPECT_THAT(rtcp_packet_parser_.pli()->num_packets(), Eq(1));
703
Philipp Hancke006206d2021-03-24 17:49:02 +0100704 // T+keyframetimeout: still no key frame received, expect key frame request
705 // sent again.
Tommiae4d0972020-05-18 08:45:38 +0200706 time_controller_.AdvanceTime(tick);
Evan Shrubsole44be5792022-04-26 16:24:41 +0200707 video_receive_stream_->OnCompleteFrame(
708 MakeFrame(VideoFrameType::kVideoFrameDelta, 2));
Evan Shrubsolea4062722022-05-02 17:46:07 +0200709 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
Tommiae4d0972020-05-18 08:45:38 +0200710 testing::Mock::VerifyAndClearExpectations(&mock_transport_);
711
philipel27b35a72022-07-05 09:59:55 +0200712 EXPECT_THAT(rtcp_packet_parser_.pli()->num_packets(), Eq(2));
713
Philipp Hancke006206d2021-03-24 17:49:02 +0100714 // T+keyframetimeout: now send a key frame - we should not observe new key
715 // frame requests after this.
Evan Shrubsole44be5792022-04-26 16:24:41 +0200716 video_receive_stream_->OnCompleteFrame(
717 MakeFrame(VideoFrameType::kVideoFrameKey, 3));
Evan Shrubsolea4062722022-05-02 17:46:07 +0200718 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
Tommiae4d0972020-05-18 08:45:38 +0200719 time_controller_.AdvanceTime(2 * tick);
Evan Shrubsole44be5792022-04-26 16:24:41 +0200720 video_receive_stream_->OnCompleteFrame(
721 MakeFrame(VideoFrameType::kVideoFrameDelta, 4));
Evan Shrubsolea4062722022-05-02 17:46:07 +0200722 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
philipel27b35a72022-07-05 09:59:55 +0200723
724 EXPECT_THAT(rtcp_packet_parser_.pli()->num_packets(), Eq(2));
Tommiae4d0972020-05-18 08:45:38 +0200725}
726
Evan Shrubsolea4062722022-05-02 17:46:07 +0200727TEST_P(VideoReceiveStream2Test,
Markus Handell588f9b32021-04-08 19:19:50 +0200728 DispatchesEncodedFrameSequenceStartingWithKeyframeWithoutResolution) {
Evan Shrubsole44be5792022-04-26 16:24:41 +0200729 video_receive_stream_->Start();
Markus Handell588f9b32021-04-08 19:19:50 +0200730 testing::MockFunction<void(const RecordableEncodedFrame&)> callback;
Evan Shrubsole44be5792022-04-26 16:24:41 +0200731 video_receive_stream_->SetAndGetRecordingState(
Tommif6f45432022-05-20 15:21:20 +0200732 VideoReceiveStreamInterface::RecordingState(callback.AsStdFunction()),
Markus Handell588f9b32021-04-08 19:19:50 +0200733 /*generate_key_frame=*/false);
734
735 InSequence s;
Jonas Oreland0deda152022-09-23 12:08:57 +0200736 EXPECT_CALL(callback,
737 Call(MatchResolution(test::FakeDecoder::kDefaultWidth,
738 test::FakeDecoder::kDefaultHeight)));
Markus Handell588f9b32021-04-08 19:19:50 +0200739 EXPECT_CALL(callback, Call);
740
Evan Shrubsole44be5792022-04-26 16:24:41 +0200741 video_receive_stream_->OnCompleteFrame(
Markus Handell588f9b32021-04-08 19:19:50 +0200742 MakeFrameWithResolution(VideoFrameType::kVideoFrameKey, 0, 0, 0));
Evan Shrubsolea4062722022-05-02 17:46:07 +0200743 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
Evan Shrubsole44be5792022-04-26 16:24:41 +0200744 video_receive_stream_->OnCompleteFrame(
Markus Handell588f9b32021-04-08 19:19:50 +0200745 MakeFrameWithResolution(VideoFrameType::kVideoFrameDelta, 1, 0, 0));
Evan Shrubsolea4062722022-05-02 17:46:07 +0200746 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
Markus Handell588f9b32021-04-08 19:19:50 +0200747
Evan Shrubsole44be5792022-04-26 16:24:41 +0200748 video_receive_stream_->Stop();
Markus Handell588f9b32021-04-08 19:19:50 +0200749}
750
Evan Shrubsolea4062722022-05-02 17:46:07 +0200751TEST_P(VideoReceiveStream2Test,
Markus Handell588f9b32021-04-08 19:19:50 +0200752 DispatchesEncodedFrameSequenceStartingWithKeyframeWithResolution) {
Evan Shrubsole44be5792022-04-26 16:24:41 +0200753 video_receive_stream_->Start();
Markus Handell588f9b32021-04-08 19:19:50 +0200754 testing::MockFunction<void(const RecordableEncodedFrame&)> callback;
Evan Shrubsole44be5792022-04-26 16:24:41 +0200755 video_receive_stream_->SetAndGetRecordingState(
Tommif6f45432022-05-20 15:21:20 +0200756 VideoReceiveStreamInterface::RecordingState(callback.AsStdFunction()),
Markus Handell588f9b32021-04-08 19:19:50 +0200757 /*generate_key_frame=*/false);
758
759 InSequence s;
Jonas Oreland0deda152022-09-23 12:08:57 +0200760 EXPECT_CALL(callback, Call(MatchResolution(1080u, 720u)));
Markus Handell588f9b32021-04-08 19:19:50 +0200761 EXPECT_CALL(callback, Call);
762
Evan Shrubsole44be5792022-04-26 16:24:41 +0200763 video_receive_stream_->OnCompleteFrame(
Markus Handell588f9b32021-04-08 19:19:50 +0200764 MakeFrameWithResolution(VideoFrameType::kVideoFrameKey, 0, 1080, 720));
Evan Shrubsolea4062722022-05-02 17:46:07 +0200765 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
Evan Shrubsole44be5792022-04-26 16:24:41 +0200766 video_receive_stream_->OnCompleteFrame(
Markus Handell588f9b32021-04-08 19:19:50 +0200767 MakeFrameWithResolution(VideoFrameType::kVideoFrameDelta, 1, 0, 0));
Evan Shrubsolea4062722022-05-02 17:46:07 +0200768 EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
Markus Handell588f9b32021-04-08 19:19:50 +0200769
Evan Shrubsole44be5792022-04-26 16:24:41 +0200770 video_receive_stream_->Stop();
Markus Handell588f9b32021-04-08 19:19:50 +0200771}
772
Evan Shrubsolea4062722022-05-02 17:46:07 +0200773TEST_P(VideoReceiveStream2Test, DependantFramesAreScheduled) {
774 video_receive_stream_->Start();
775
776 auto key_frame = test::FakeFrameBuilder()
777 .Id(0)
778 .PayloadType(99)
779 .Time(kFirstRtpTimestamp)
780 .ReceivedTime(kStartTime)
781 .AsLast()
782 .Build();
783 auto delta_frame = test::FakeFrameBuilder()
784 .Id(1)
785 .PayloadType(99)
786 .Time(RtpTimestampForFrame(1))
787 .ReceivedTime(ReceiveTimeForFrame(1))
788 .Refs({0})
789 .AsLast()
790 .Build();
791
792 // Expect frames are decoded in order.
793 InSequence seq;
794 EXPECT_CALL(mock_decoder_,
795 Decode(test::RtpTimestamp(kFirstRtpTimestamp), _, _));
796 EXPECT_CALL(mock_decoder_, Decode(test::RtpTimestamp(kFirstRtpTimestamp +
797 k30FpsRtpTimestampDelta),
798 _, _))
799 .Times(1);
800 video_receive_stream_->OnCompleteFrame(std::move(key_frame));
801 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
802
803 time_controller_.AdvanceTime(k30FpsDelay);
804 video_receive_stream_->OnCompleteFrame(std::move(delta_frame));
805 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame());
806
807 video_receive_stream_->Stop();
808}
809
810TEST_P(VideoReceiveStream2Test, FramesScheduledInOrder) {
811 video_receive_stream_->Start();
812
813 auto key_frame = test::FakeFrameBuilder()
814 .Id(0)
815 .PayloadType(99)
816 .Time(kFirstRtpTimestamp)
817 .AsLast()
818 .Build();
819 auto delta_frame1 = test::FakeFrameBuilder()
820 .Id(1)
821 .PayloadType(99)
822 .Time(RtpTimestampForFrame(1))
823 .Refs({0})
824 .AsLast()
825 .Build();
826 auto delta_frame2 = test::FakeFrameBuilder()
827 .Id(2)
828 .PayloadType(99)
829 .Time(RtpTimestampForFrame(2))
830 .Refs({1})
831 .AsLast()
832 .Build();
833
834 // Expect frames are decoded in order despite delta_frame1 arriving first.
835 InSequence seq;
836 EXPECT_CALL(mock_decoder_,
837 Decode(test::RtpTimestamp(kFirstRtpTimestamp), _, _))
838 .Times(1);
839 EXPECT_CALL(mock_decoder_,
840 Decode(test::RtpTimestamp(RtpTimestampForFrame(1)), _, _))
841 .Times(1);
842 EXPECT_CALL(mock_decoder_,
843 Decode(test::RtpTimestamp(RtpTimestampForFrame(2)), _, _))
844 .Times(1);
845 key_frame->SetReceivedTime(clock_->CurrentTime().ms());
846 video_receive_stream_->OnCompleteFrame(std::move(key_frame));
847 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
848
849 delta_frame2->SetReceivedTime(clock_->CurrentTime().ms());
850 video_receive_stream_->OnCompleteFrame(std::move(delta_frame2));
851 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), DidNotReceiveFrame());
852 // `delta_frame1` arrives late.
853 delta_frame1->SetReceivedTime(clock_->CurrentTime().ms());
854 video_receive_stream_->OnCompleteFrame(std::move(delta_frame1));
855 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame());
856 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay * 2), RenderedFrame());
857 video_receive_stream_->Stop();
858}
859
860TEST_P(VideoReceiveStream2Test, WaitsforAllSpatialLayers) {
861 video_receive_stream_->Start();
862 auto sl0 = test::FakeFrameBuilder()
863 .Id(0)
864 .PayloadType(99)
865 .Time(kFirstRtpTimestamp)
866 .ReceivedTime(kStartTime)
867 .Build();
868 auto sl1 = test::FakeFrameBuilder()
869 .Id(1)
870 .PayloadType(99)
871 .ReceivedTime(kStartTime)
872 .Time(kFirstRtpTimestamp)
873 .Refs({0})
874 .Build();
875 auto sl2 = test::FakeFrameBuilder()
876 .Id(2)
877 .PayloadType(99)
878 .ReceivedTime(kStartTime)
879 .Time(kFirstRtpTimestamp)
880 .Refs({0, 1})
881 .AsLast()
882 .Build();
883
884 // No decodes should be called until `sl2` is received.
885 EXPECT_CALL(mock_decoder_, Decode).Times(0);
886 sl0->SetReceivedTime(clock_->CurrentTime().ms());
887 video_receive_stream_->OnCompleteFrame(std::move(sl0));
888 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()),
889 DidNotReceiveFrame());
890 video_receive_stream_->OnCompleteFrame(std::move(sl1));
891 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()),
892 DidNotReceiveFrame());
893 // When `sl2` arrives decode should happen.
894 EXPECT_CALL(mock_decoder_,
895 Decode(test::RtpTimestamp(kFirstRtpTimestamp), _, _))
896 .Times(1);
897 video_receive_stream_->OnCompleteFrame(std::move(sl2));
898 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
899 video_receive_stream_->Stop();
900}
901
902TEST_P(VideoReceiveStream2Test, FramesFastForwardOnSystemHalt) {
903 video_receive_stream_->Start();
904
905 // The frame structure looks like this,
906 // F1
907 // /
908 // F0 --> F2
909 //
910 // In this case we will have a system halt simulated. By the time the system
911 // resumes, F1 will be old and so F2 should be decoded.
912 auto key_frame = test::FakeFrameBuilder()
913 .Id(0)
914 .PayloadType(99)
915 .Time(kFirstRtpTimestamp)
916 .AsLast()
917 .Build();
918 auto ffwd_frame = test::FakeFrameBuilder()
919 .Id(1)
920 .PayloadType(99)
921 .Time(RtpTimestampForFrame(1))
922 .Refs({0})
923 .AsLast()
924 .Build();
925 auto rendered_frame = test::FakeFrameBuilder()
926 .Id(2)
927 .PayloadType(99)
928 .Time(RtpTimestampForFrame(2))
929 .Refs({0})
930 .AsLast()
931 .Build();
932 InSequence seq;
933 EXPECT_CALL(mock_decoder_,
934 Decode(test::RtpTimestamp(kFirstRtpTimestamp), _, _))
philipel27b35a72022-07-05 09:59:55 +0200935 .WillOnce(testing::DoAll(Invoke([&] {
Evan Shrubsolea4062722022-05-02 17:46:07 +0200936 // System halt will be simulated in the decode.
937 time_controller_.AdvanceTime(k30FpsDelay * 2);
938 }),
939 DefaultDecodeAction()));
940 EXPECT_CALL(mock_decoder_,
941 Decode(test::RtpTimestamp(RtpTimestampForFrame(2)), _, _));
942 video_receive_stream_->OnCompleteFrame(std::move(key_frame));
943 video_receive_stream_->OnCompleteFrame(std::move(ffwd_frame));
944 video_receive_stream_->OnCompleteFrame(std::move(rendered_frame));
Evan Shrubsole7cbd8de2022-08-16 08:08:53 +0000945 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()),
946 RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(0))));
947 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()),
948 RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(2))));
Evan Shrubsolea4062722022-05-02 17:46:07 +0200949
950 // Check stats show correct dropped frames.
951 auto stats = video_receive_stream_->GetStats();
952 EXPECT_EQ(stats.frames_dropped, 1u);
953
954 video_receive_stream_->Stop();
955}
956
957TEST_P(VideoReceiveStream2Test, BetterFrameInsertedWhileWaitingToDecodeFrame) {
958 video_receive_stream_->Start();
959
960 auto key_frame = test::FakeFrameBuilder()
961 .Id(0)
962 .PayloadType(99)
963 .Time(kFirstRtpTimestamp)
964 .ReceivedTime(ReceiveTimeForFrame(0))
965 .AsLast()
966 .Build();
967 auto f1 = test::FakeFrameBuilder()
968 .Id(1)
969 .PayloadType(99)
970 .Time(RtpTimestampForFrame(1))
971 .ReceivedTime(ReceiveTimeForFrame(1))
972 .Refs({0})
973 .AsLast()
974 .Build();
975 auto f2 = test::FakeFrameBuilder()
976 .Id(2)
977 .PayloadType(99)
978 .Time(RtpTimestampForFrame(2))
979 .ReceivedTime(ReceiveTimeForFrame(2))
980 .Refs({0})
981 .AsLast()
982 .Build();
983
984 video_receive_stream_->OnCompleteFrame(std::move(key_frame));
985 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
986
987 InSequence seq;
988 EXPECT_CALL(mock_decoder_,
989 Decode(test::RtpTimestamp(RtpTimestampForFrame(1)), _, _))
990 .Times(1);
991 EXPECT_CALL(mock_decoder_,
992 Decode(test::RtpTimestamp(RtpTimestampForFrame(2)), _, _))
993 .Times(1);
994 // Simulate f1 arriving after f2 but before f2 is decoded.
995 video_receive_stream_->OnCompleteFrame(std::move(f2));
996 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), DidNotReceiveFrame());
997 video_receive_stream_->OnCompleteFrame(std::move(f1));
998 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame());
999 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame());
1000
1001 video_receive_stream_->Stop();
1002}
1003
1004// Note: This test takes a long time (~10s) to run if the fake metronome is
1005// active. Since the test needs to wait for the timestamp to rollover, it has a
1006// fake delay of around 6.5 hours. Even though time is simulated, this will be
1007// around 1,500,000 metronome tick invocations.
1008TEST_P(VideoReceiveStream2Test, RtpTimestampWrapAround) {
Evan Shrubsole7cbd8de2022-08-16 08:08:53 +00001009 EXPECT_CALL(mock_transport_, SendRtcp).Times(AnyNumber());
Evan Shrubsolea4062722022-05-02 17:46:07 +02001010 video_receive_stream_->Start();
1011
1012 constexpr uint32_t kBaseRtp = std::numeric_limits<uint32_t>::max() / 2;
1013 video_receive_stream_->OnCompleteFrame(
1014 test::FakeFrameBuilder()
1015 .Id(0)
1016 .PayloadType(99)
1017 .Time(kBaseRtp)
1018 .ReceivedTime(clock_->CurrentTime())
1019 .AsLast()
1020 .Build());
1021 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
1022 time_controller_.AdvanceTime(k30FpsDelay);
1023 video_receive_stream_->OnCompleteFrame(
1024 test::FakeFrameBuilder()
1025 .Id(1)
1026 .PayloadType(99)
1027 .Time(kBaseRtp + k30FpsRtpTimestampDelta)
1028 .ReceivedTime(clock_->CurrentTime())
1029 .AsLast()
1030 .Build());
1031 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame());
1032
1033 // Pause stream so that RTP timestamp wraps around.
1034 constexpr uint32_t kLastRtp = kBaseRtp + k30FpsRtpTimestampDelta;
1035 constexpr uint32_t kWrapAroundRtp =
1036 kLastRtp + std::numeric_limits<uint32_t>::max() / 2 + 1;
1037 // Pause for corresponding delay such that RTP timestamp would increase this
1038 // much at 30fps.
1039 constexpr TimeDelta kWrapAroundDelay =
1040 (std::numeric_limits<uint32_t>::max() / 2 + 1) / kRtpTimestampHz;
1041
1042 time_controller_.AdvanceTime(kWrapAroundDelay);
1043 video_receive_stream_->OnCompleteFrame(
1044 test::FakeFrameBuilder()
1045 .Id(2)
1046 .PayloadType(99)
1047 .Time(kWrapAroundRtp)
1048 .ReceivedTime(clock_->CurrentTime())
1049 .AsLast()
1050 .Build());
1051 EXPECT_CALL(mock_decoder_, Decode(test::RtpTimestamp(kWrapAroundRtp), _, _))
1052 .Times(1);
1053 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
1054
1055 video_receive_stream_->Stop();
1056}
1057
Evan Shrubsoledcb9c5d2022-06-13 15:39:53 +00001058// If a frame was lost causing the stream to become temporarily non-decodable
1059// and the sender reduces their framerate during this time, the video stream
1060// should start decoding at the new framerate. However, if the connection is
1061// poor, a keyframe will take a long time to send. If the timing of the incoming
1062// frames was not kept up to date with the new framerate while the stream was
1063// decodable, this late frame will have a large delay as the rtp timestamp of
1064// this keyframe will look like the frame arrived early if the frame-rate was
1065// not updated.
1066TEST_P(VideoReceiveStream2Test, PoorConnectionWithFpsChangeDuringLostFrame) {
1067 video_receive_stream_->Start();
1068
1069 constexpr Frequency k15Fps = Frequency::Hertz(15);
1070 constexpr TimeDelta k15FpsDelay = 1 / k15Fps;
1071 constexpr uint32_t k15FpsRtpTimestampDelta = kRtpTimestampHz / k15Fps;
1072
1073 // Initial keyframe and frames at 30fps.
1074 video_receive_stream_->OnCompleteFrame(
1075 test::FakeFrameBuilder()
1076 .Id(0)
1077 .PayloadType(99)
1078 .Time(RtpTimestampForFrame(0))
1079 .ReceivedTime(ReceiveTimeForFrame(0))
1080 .AsLast()
1081 .Build());
1082 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay, /*advance_time=*/true),
1083 RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(0))));
1084
1085 video_receive_stream_->OnCompleteFrame(
1086 test::FakeFrameBuilder()
1087 .Id(1)
1088 .PayloadType(99)
1089 .Time(RtpTimestampForFrame(1))
1090 .ReceivedTime(ReceiveTimeForFrame(1))
1091 .Refs({0})
1092 .AsLast()
1093 .Build());
1094 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay, /*advance_time=*/true),
1095 RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(1))));
1096
1097 // Simulate lost frame 2, followed by 2 second of frames at 30fps, followed by
1098 // 2 second of frames at 15 fps, and then a keyframe.
1099 time_controller_.AdvanceTime(k30FpsDelay);
1100
1101 Timestamp send_30fps_end_time = clock_->CurrentTime() + TimeDelta::Seconds(2);
1102 int id = 3;
1103 EXPECT_CALL(mock_transport_, SendRtcp).Times(AnyNumber());
1104 while (clock_->CurrentTime() < send_30fps_end_time) {
1105 ++id;
1106 video_receive_stream_->OnCompleteFrame(
1107 test::FakeFrameBuilder()
1108 .Id(id)
1109 .PayloadType(99)
1110 .Time(RtpTimestampForFrame(id))
1111 .ReceivedTime(ReceiveTimeForFrame(id))
1112 .Refs({id - 1})
1113 .AsLast()
1114 .Build());
1115 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay, /*advance_time=*/true),
1116 Eq(absl::nullopt));
1117 }
1118 uint32_t current_rtp = RtpTimestampForFrame(id);
1119 Timestamp send_15fps_end_time = clock_->CurrentTime() + TimeDelta::Seconds(2);
1120 while (clock_->CurrentTime() < send_15fps_end_time) {
1121 ++id;
1122 current_rtp += k15FpsRtpTimestampDelta;
1123 video_receive_stream_->OnCompleteFrame(
1124 test::FakeFrameBuilder()
1125 .Id(id)
1126 .PayloadType(99)
1127 .Time(current_rtp)
1128 .ReceivedTime(clock_->CurrentTime())
1129 .Refs({id - 1})
1130 .AsLast()
1131 .Build());
1132 EXPECT_THAT(fake_renderer_.WaitForFrame(k15FpsDelay, /*advance_time=*/true),
1133 Eq(absl::nullopt));
1134 }
1135
1136 ++id;
1137 current_rtp += k15FpsRtpTimestampDelta;
1138 // Insert keyframe which will recover the stream. However, on a poor
1139 // connection the keyframe will take significant time to send.
1140 constexpr TimeDelta kKeyframeDelay = TimeDelta::Millis(200);
1141 video_receive_stream_->OnCompleteFrame(
1142 test::FakeFrameBuilder()
1143 .Id(id)
1144 .PayloadType(99)
1145 .Time(current_rtp)
1146 .ReceivedTime(clock_->CurrentTime() + kKeyframeDelay)
1147 .AsLast()
1148 .Build());
1149 // If the framerate was not updated to be 15fps from the frames that arrived
1150 // previously, this will fail, as the delay will be longer.
1151 EXPECT_THAT(fake_renderer_.WaitForFrame(k15FpsDelay, /*advance_time=*/true),
1152 RenderedFrameWith(RtpTimestamp(current_rtp)));
1153
1154 video_receive_stream_->Stop();
1155}
1156
Evan Shrubsole4d3ba772022-06-22 14:32:36 +00001157TEST_P(VideoReceiveStream2Test, StreamShouldNotTimeoutWhileWaitingForFrame) {
1158 // Disable smoothing since this makes it hard to test frame timing.
1159 config_.enable_prerenderer_smoothing = false;
1160 RecreateReceiveStream();
1161
1162 video_receive_stream_->Start();
1163 EXPECT_CALL(mock_transport_, SendRtcp).Times(AnyNumber());
1164
1165 video_receive_stream_->OnCompleteFrame(
1166 test::FakeFrameBuilder()
1167 .Id(0)
1168 .PayloadType(99)
1169 .Time(RtpTimestampForFrame(0))
1170 .ReceivedTime(ReceiveTimeForFrame(0))
1171 .AsLast()
1172 .Build());
1173 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay, /*advance_time=*/true),
1174 RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(0))));
1175
1176 for (int id = 1; id < 30; ++id) {
1177 video_receive_stream_->OnCompleteFrame(
1178 test::FakeFrameBuilder()
1179 .Id(id)
1180 .PayloadType(99)
1181 .Time(RtpTimestampForFrame(id))
1182 .ReceivedTime(ReceiveTimeForFrame(id))
1183 .Refs({0})
1184 .AsLast()
1185 .Build());
1186 EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay, /*advance_time=*/true),
1187 RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(id))));
1188 }
1189
1190 // Simulate a pause in the stream, followed by a decodable frame that is ready
1191 // long in the future. The stream should not timeout in this case, but rather
1192 // decode the frame just before the timeout.
1193 time_controller_.AdvanceTime(TimeDelta::Millis(2900));
1194 uint32_t late_decode_rtp = kFirstRtpTimestamp + 200 * k30FpsRtpTimestampDelta;
1195 video_receive_stream_->OnCompleteFrame(
1196 test::FakeFrameBuilder()
1197 .Id(121)
1198 .PayloadType(99)
1199 .Time(late_decode_rtp)
1200 .ReceivedTime(clock_->CurrentTime())
1201 .AsLast()
1202 .Build());
1203 EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Millis(100),
1204 /*advance_time=*/true),
1205 RenderedFrameWith(RtpTimestamp(late_decode_rtp)));
1206
1207 video_receive_stream_->Stop();
1208}
1209
Evan Shrubsolea4062722022-05-02 17:46:07 +02001210INSTANTIATE_TEST_SUITE_P(VideoReceiveStream2Test,
1211 VideoReceiveStream2Test,
1212 testing::Bool(),
1213 [](const auto& test_param_info) {
Evan Shrubsoledcb9c5d2022-06-13 15:39:53 +00001214 return (test_param_info.param
1215 ? "ScheduleDecodesWithMetronome"
1216 : "ScheduleDecodesWithPostTask");
Evan Shrubsolea4062722022-05-02 17:46:07 +02001217 });
1218
Tommiae4d0972020-05-18 08:45:38 +02001219} // namespace webrtc