blob: c05fe089c5c93b8a2faf40409966cb54e2b7c484 [file] [log] [blame]
philipelbe7a9e52016-05-19 12:19:35 +02001/*
2 * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "modules/video_coding/frame_buffer2.h"
philipelbe7a9e52016-05-19 12:19:35 +020012
13#include <algorithm>
14#include <cstring>
15#include <limits>
Mirko Bonadei317a1f02019-09-17 17:06:18 +020016#include <memory>
philipelbe7a9e52016-05-19 12:19:35 +020017#include <vector>
18
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020019#include "modules/video_coding/frame_object.h"
20#include "modules/video_coding/jitter_estimator.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020021#include "modules/video_coding/timing.h"
Bjorn Tereliusa194e582017-10-25 13:07:09 +020022#include "rtc_base/numerics/sequence_number_util.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020023#include "rtc_base/platform_thread.h"
24#include "rtc_base/random.h"
25#include "system_wrappers/include/clock.h"
Niels Möller7cca0422019-04-29 16:12:19 +020026#include "test/field_trial.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020027#include "test/gmock.h"
28#include "test/gtest.h"
philipel19053972020-01-29 17:36:11 +010029#include "test/time_controller/simulated_time_controller.h"
philipelbe7a9e52016-05-19 12:19:35 +020030
Mirko Bonadei6a489f22019-04-09 15:11:12 +020031using ::testing::_;
32using ::testing::Return;
philipela45102f2017-02-22 05:30:39 -080033
philipelbe7a9e52016-05-19 12:19:35 +020034namespace webrtc {
35namespace video_coding {
36
37class VCMTimingFake : public VCMTiming {
38 public:
39 explicit VCMTimingFake(Clock* clock) : VCMTiming(clock) {}
40
41 int64_t RenderTimeMs(uint32_t frame_timestamp,
42 int64_t now_ms) const override {
43 if (last_ms_ == -1) {
44 last_ms_ = now_ms + kDelayMs;
45 last_timestamp_ = frame_timestamp;
46 }
47
48 uint32_t diff = MinDiff(frame_timestamp, last_timestamp_);
49 if (AheadOf(frame_timestamp, last_timestamp_))
50 last_ms_ += diff / 90;
51 else
52 last_ms_ -= diff / 90;
53
54 last_timestamp_ = frame_timestamp;
55 return last_ms_;
56 }
57
Ilya Nikolaevskiy8c4fe162018-02-27 15:49:47 +010058 int64_t MaxWaitingTime(int64_t render_time_ms,
59 int64_t now_ms) const override {
60 return render_time_ms - now_ms - kDecodeTime;
philipelbe7a9e52016-05-19 12:19:35 +020061 }
62
Johannes Kronbfd343b2019-07-01 10:07:50 +020063 bool GetTimings(int* max_decode_ms,
philipela45102f2017-02-22 05:30:39 -080064 int* current_delay_ms,
65 int* target_delay_ms,
66 int* jitter_buffer_ms,
67 int* min_playout_delay_ms,
68 int* render_delay_ms) const override {
69 return true;
70 }
71
Niels Möller7cca0422019-04-29 16:12:19 +020072 int GetCurrentJitter() {
Niels Möller7cca0422019-04-29 16:12:19 +020073 int max_decode_ms;
74 int current_delay_ms;
75 int target_delay_ms;
76 int jitter_buffer_ms;
77 int min_playout_delay_ms;
78 int render_delay_ms;
Johannes Kronbfd343b2019-07-01 10:07:50 +020079 VCMTiming::GetTimings(&max_decode_ms, &current_delay_ms, &target_delay_ms,
80 &jitter_buffer_ms, &min_playout_delay_ms,
81 &render_delay_ms);
Niels Möller7cca0422019-04-29 16:12:19 +020082 return jitter_buffer_ms;
83 }
84
philipelbe7a9e52016-05-19 12:19:35 +020085 private:
86 static constexpr int kDelayMs = 50;
87 static constexpr int kDecodeTime = kDelayMs / 2;
88 mutable uint32_t last_timestamp_ = 0;
89 mutable int64_t last_ms_ = -1;
90};
91
philipele7c891f2018-02-22 14:35:06 +010092class FrameObjectFake : public EncodedFrame {
philipelbe7a9e52016-05-19 12:19:35 +020093 public:
philipelb4d31082016-07-11 08:46:29 -070094 int64_t ReceivedTime() const override { return 0; }
95
96 int64_t RenderTime() const override { return _renderTimeMs; }
Niels Möller7cca0422019-04-29 16:12:19 +020097
98 bool delayed_by_retransmission() const override {
99 return delayed_by_retransmission_;
100 }
101 void set_delayed_by_retransmission(bool delayed) {
102 delayed_by_retransmission_ = delayed;
103 }
104
105 private:
106 bool delayed_by_retransmission_ = false;
philipela45102f2017-02-22 05:30:39 -0800107};
108
109class VCMReceiveStatisticsCallbackMock : public VCMReceiveStatisticsCallback {
110 public:
Danil Chapovalovf2c0f152020-05-19 13:20:27 +0200111 MOCK_METHOD(void,
112 OnCompleteFrame,
113 (bool is_keyframe,
114 size_t size_bytes,
115 VideoContentType content_type),
116 (override));
117 MOCK_METHOD(void, OnDroppedFrames, (uint32_t frames_dropped), (override));
118 MOCK_METHOD(void,
119 OnFrameBufferTimingsUpdated,
120 (int max_decode_ms,
121 int current_delay_ms,
122 int target_delay_ms,
123 int jitter_buffer_ms,
124 int min_playout_delay_ms,
125 int render_delay_ms),
126 (override));
127 MOCK_METHOD(void,
128 OnTimingFrameInfoUpdated,
129 (const TimingFrameInfo& info),
130 (override));
philipelbe7a9e52016-05-19 12:19:35 +0200131};
132
133class TestFrameBuffer2 : public ::testing::Test {
134 protected:
135 static constexpr int kMaxReferences = 5;
136 static constexpr int kFps1 = 1000;
137 static constexpr int kFps10 = kFps1 / 10;
138 static constexpr int kFps20 = kFps1 / 20;
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100139 static constexpr size_t kFrameSize = 10;
philipelbe7a9e52016-05-19 12:19:35 +0200140
141 TestFrameBuffer2()
Niels Möller7cca0422019-04-29 16:12:19 +0200142 : trial_("WebRTC-AddRttToPlayoutDelay/Enabled/"),
Danil Chapovalov55284022020-02-07 14:53:52 +0100143 time_controller_(Timestamp::Seconds(0)),
philipel19053972020-01-29 17:36:11 +0100144 time_task_queue_(
145 time_controller_.GetTaskQueueFactory()->CreateTaskQueue(
146 "extract queue",
147 TaskQueueFactory::Priority::NORMAL)),
148 timing_(time_controller_.GetClock()),
149 buffer_(new FrameBuffer(time_controller_.GetClock(),
150 &timing_,
151 &stats_callback_)),
152 rand_(0x34678213) {}
philipelbe7a9e52016-05-19 12:19:35 +0200153
154 template <typename... T>
Niels Möller7cca0422019-04-29 16:12:19 +0200155 std::unique_ptr<FrameObjectFake> CreateFrame(uint16_t picture_id,
156 uint8_t spatial_layer,
157 int64_t ts_ms,
158 bool inter_layer_predicted,
159 bool last_spatial_layer,
Sergey Silkin2799e632019-05-17 09:51:39 +0200160 size_t frame_size_bytes,
Niels Möller7cca0422019-04-29 16:12:19 +0200161 T... refs) {
philipelbe7a9e52016-05-19 12:19:35 +0200162 static_assert(sizeof...(refs) <= kMaxReferences,
philipele7c891f2018-02-22 14:35:06 +0100163 "To many references specified for EncodedFrame.");
kwiberg5b9746e2017-08-16 04:52:35 -0700164 std::array<uint16_t, sizeof...(refs)> references = {
165 {rtc::checked_cast<uint16_t>(refs)...}};
philipelbe7a9e52016-05-19 12:19:35 +0200166
Mirko Bonadei317a1f02019-09-17 17:06:18 +0200167 auto frame = std::make_unique<FrameObjectFake>();
philipel0fa82a62018-03-19 15:34:53 +0100168 frame->id.picture_id = picture_id;
169 frame->id.spatial_layer = spatial_layer;
Sergey Silkin61832dd2018-12-20 14:32:14 +0100170 frame->SetSpatialIndex(spatial_layer);
Niels Möller23775882018-08-16 10:24:12 +0200171 frame->SetTimestamp(ts_ms * 90);
philipelbe7a9e52016-05-19 12:19:35 +0200172 frame->num_references = references.size();
173 frame->inter_layer_predicted = inter_layer_predicted;
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100174 frame->is_last_spatial_layer = last_spatial_layer;
175 // Add some data to buffer.
Niels Möllerb9bfe652019-10-03 08:43:53 +0200176 frame->SetEncodedData(EncodedImageBuffer::Create(frame_size_bytes));
philipelbe7a9e52016-05-19 12:19:35 +0200177 for (size_t r = 0; r < references.size(); ++r)
178 frame->references[r] = references[r];
Niels Möller7cca0422019-04-29 16:12:19 +0200179 return frame;
180 }
philipelbe7a9e52016-05-19 12:19:35 +0200181
Niels Möller7cca0422019-04-29 16:12:19 +0200182 template <typename... T>
183 int InsertFrame(uint16_t picture_id,
184 uint8_t spatial_layer,
185 int64_t ts_ms,
186 bool inter_layer_predicted,
187 bool last_spatial_layer,
Sergey Silkin2799e632019-05-17 09:51:39 +0200188 size_t frame_size_bytes,
Niels Möller7cca0422019-04-29 16:12:19 +0200189 T... refs) {
Sergey Silkin2799e632019-05-17 09:51:39 +0200190 return buffer_->InsertFrame(
191 CreateFrame(picture_id, spatial_layer, ts_ms, inter_layer_predicted,
192 last_spatial_layer, frame_size_bytes, refs...));
Niels Möller7cca0422019-04-29 16:12:19 +0200193 }
194
195 int InsertNackedFrame(uint16_t picture_id, int64_t ts_ms) {
196 std::unique_ptr<FrameObjectFake> frame =
Sergey Silkin2799e632019-05-17 09:51:39 +0200197 CreateFrame(picture_id, 0, ts_ms, false, true, kFrameSize);
Niels Möller7cca0422019-04-29 16:12:19 +0200198 frame->set_delayed_by_retransmission(true);
Stefan Holmer812ceaf2018-05-15 13:00:10 +0200199 return buffer_->InsertFrame(std::move(frame));
philipelbe7a9e52016-05-19 12:19:35 +0200200 }
201
philipel3042c2d2017-08-18 04:55:02 -0700202 void ExtractFrame(int64_t max_wait_time = 0, bool keyframe_required = false) {
philipel19053972020-01-29 17:36:11 +0100203 time_task_queue_.PostTask([this, max_wait_time, keyframe_required]() {
204 buffer_->NextFrame(
205 max_wait_time, keyframe_required, &time_task_queue_,
206 [this](std::unique_ptr<video_coding::EncodedFrame> frame,
207 video_coding::FrameBuffer::ReturnReason reason) {
208 if (reason != FrameBuffer::ReturnReason::kStopped) {
209 frames_.emplace_back(std::move(frame));
210 }
211 });
212 });
philipelbe7a9e52016-05-19 12:19:35 +0200213 if (max_wait_time == 0) {
Danil Chapovalov55284022020-02-07 14:53:52 +0100214 time_controller_.AdvanceTime(TimeDelta::Millis(0));
philipelbe7a9e52016-05-19 12:19:35 +0200215 }
216 }
217
218 void CheckFrame(size_t index, int picture_id, int spatial_layer) {
philipelbe7a9e52016-05-19 12:19:35 +0200219 ASSERT_LT(index, frames_.size());
220 ASSERT_TRUE(frames_[index]);
philipel0fa82a62018-03-19 15:34:53 +0100221 ASSERT_EQ(picture_id, frames_[index]->id.picture_id);
222 ASSERT_EQ(spatial_layer, frames_[index]->id.spatial_layer);
philipelbe7a9e52016-05-19 12:19:35 +0200223 }
224
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100225 void CheckFrameSize(size_t index, size_t size) {
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100226 ASSERT_LT(index, frames_.size());
227 ASSERT_TRUE(frames_[index]);
228 ASSERT_EQ(frames_[index]->size(), size);
229 }
230
philipelbe7a9e52016-05-19 12:19:35 +0200231 void CheckNoFrame(size_t index) {
philipelbe7a9e52016-05-19 12:19:35 +0200232 ASSERT_LT(index, frames_.size());
233 ASSERT_FALSE(frames_[index]);
234 }
235
philipelbe7a9e52016-05-19 12:19:35 +0200236 uint32_t Rand() { return rand_.Rand<uint32_t>(); }
237
Niels Möller7cca0422019-04-29 16:12:19 +0200238 // The ProtectionMode tests depends on rtt-multiplier experiment.
239 test::ScopedFieldTrials trial_;
philipel19053972020-01-29 17:36:11 +0100240 webrtc::GlobalSimulatedTimeController time_controller_;
241 rtc::TaskQueue time_task_queue_;
philipelbe7a9e52016-05-19 12:19:35 +0200242 VCMTimingFake timing_;
Stefan Holmer812ceaf2018-05-15 13:00:10 +0200243 std::unique_ptr<FrameBuffer> buffer_;
philipele7c891f2018-02-22 14:35:06 +0100244 std::vector<std::unique_ptr<EncodedFrame>> frames_;
philipelbe7a9e52016-05-19 12:19:35 +0200245 Random rand_;
philipela45102f2017-02-22 05:30:39 -0800246 ::testing::NiceMock<VCMReceiveStatisticsCallbackMock> stats_callback_;
philipelbe7a9e52016-05-19 12:19:35 +0200247};
248
Sergey Silkin2799e632019-05-17 09:51:39 +0200249// From https://en.cppreference.com/w/cpp/language/static: "If ... a constexpr
250// static data member (since C++11) is odr-used, a definition at namespace scope
251// is still required... This definition is deprecated for constexpr data members
252// since C++17."
253// kFrameSize is odr-used since it is passed by reference to EXPECT_EQ().
254#if __cplusplus < 201703L
255constexpr size_t TestFrameBuffer2::kFrameSize;
256#endif
257
philipelbe7a9e52016-05-19 12:19:35 +0200258TEST_F(TestFrameBuffer2, WaitForFrame) {
259 uint16_t pid = Rand();
260 uint32_t ts = Rand();
261
philipel6e8224f2016-05-19 17:07:44 +0200262 ExtractFrame(50);
Sergey Silkin2799e632019-05-17 09:51:39 +0200263 InsertFrame(pid, 0, ts, false, true, kFrameSize);
Danil Chapovalov55284022020-02-07 14:53:52 +0100264 time_controller_.AdvanceTime(TimeDelta::Millis(50));
philipelbe7a9e52016-05-19 12:19:35 +0200265 CheckFrame(0, pid, 0);
266}
267
268TEST_F(TestFrameBuffer2, OneSuperFrame) {
269 uint16_t pid = Rand();
270 uint32_t ts = Rand();
271
Sergey Silkin2799e632019-05-17 09:51:39 +0200272 InsertFrame(pid, 0, ts, false, false, kFrameSize);
273 InsertFrame(pid, 1, ts, true, true, kFrameSize);
philipele0b2f152016-09-28 10:23:49 +0200274 ExtractFrame();
275
Sergey Silkin61832dd2018-12-20 14:32:14 +0100276 CheckFrame(0, pid, 1);
philipele0b2f152016-09-28 10:23:49 +0200277}
278
Stefan Holmer812ceaf2018-05-15 13:00:10 +0200279TEST_F(TestFrameBuffer2, ZeroPlayoutDelay) {
philipel19053972020-01-29 17:36:11 +0100280 VCMTiming timing(time_controller_.GetClock());
281 buffer_.reset(
282 new FrameBuffer(time_controller_.GetClock(), &timing, &stats_callback_));
Niels Möllerd381eed2020-09-02 15:34:40 +0200283 const VideoPlayoutDelay kPlayoutDelayMs = {0, 0};
Stefan Holmer812ceaf2018-05-15 13:00:10 +0200284 std::unique_ptr<FrameObjectFake> test_frame(new FrameObjectFake());
285 test_frame->id.picture_id = 0;
286 test_frame->SetPlayoutDelay(kPlayoutDelayMs);
287 buffer_->InsertFrame(std::move(test_frame));
288 ExtractFrame(0, false);
289 CheckFrame(0, 0, 0);
290 EXPECT_EQ(0, frames_[0]->RenderTimeMs());
291}
292
aleloi3e005282017-01-26 05:38:00 -0800293// Flaky test, see bugs.webrtc.org/7068.
294TEST_F(TestFrameBuffer2, DISABLED_OneUnorderedSuperFrame) {
philipele0b2f152016-09-28 10:23:49 +0200295 uint16_t pid = Rand();
296 uint32_t ts = Rand();
297
philipel6e8224f2016-05-19 17:07:44 +0200298 ExtractFrame(50);
Sergey Silkin2799e632019-05-17 09:51:39 +0200299 InsertFrame(pid, 1, ts, true, true, kFrameSize);
300 InsertFrame(pid, 0, ts, false, false, kFrameSize);
Danil Chapovalov55284022020-02-07 14:53:52 +0100301 time_controller_.AdvanceTime(TimeDelta::Millis(0));
philipelbe7a9e52016-05-19 12:19:35 +0200302
303 CheckFrame(0, pid, 0);
304 CheckFrame(1, pid, 1);
305}
306
philipel94616b32016-05-20 10:43:01 +0200307TEST_F(TestFrameBuffer2, DISABLED_OneLayerStreamReordered) {
philipel6e8224f2016-05-19 17:07:44 +0200308 uint16_t pid = Rand();
309 uint32_t ts = Rand();
310
Sergey Silkin2799e632019-05-17 09:51:39 +0200311 InsertFrame(pid, 0, ts, false, true, kFrameSize);
philipel6e8224f2016-05-19 17:07:44 +0200312 ExtractFrame();
313 CheckFrame(0, pid, 0);
314 for (int i = 1; i < 10; i += 2) {
315 ExtractFrame(50);
Sergey Silkin2799e632019-05-17 09:51:39 +0200316 InsertFrame(pid + i + 1, 0, ts + (i + 1) * kFps10, false, true, kFrameSize,
317 pid + i);
Danil Chapovalov55284022020-02-07 14:53:52 +0100318 time_controller_.AdvanceTime(TimeDelta::Millis(kFps10));
Sergey Silkin2799e632019-05-17 09:51:39 +0200319 InsertFrame(pid + i, 0, ts + i * kFps10, false, true, kFrameSize,
320 pid + i - 1);
Danil Chapovalov55284022020-02-07 14:53:52 +0100321 time_controller_.AdvanceTime(TimeDelta::Millis(kFps10));
philipel6e8224f2016-05-19 17:07:44 +0200322 ExtractFrame();
323 CheckFrame(i, pid + i, 0);
324 CheckFrame(i + 1, pid + i + 1, 0);
325 }
326}
philipel6e8224f2016-05-19 17:07:44 +0200327
328TEST_F(TestFrameBuffer2, ExtractFromEmptyBuffer) {
329 ExtractFrame();
330 CheckNoFrame(0);
331}
332
philipel93e451b2016-10-06 12:25:13 +0200333TEST_F(TestFrameBuffer2, MissingFrame) {
334 uint16_t pid = Rand();
335 uint32_t ts = Rand();
336
Sergey Silkin2799e632019-05-17 09:51:39 +0200337 InsertFrame(pid, 0, ts, false, true, kFrameSize);
338 InsertFrame(pid + 2, 0, ts, false, true, kFrameSize, pid);
339 InsertFrame(pid + 3, 0, ts, false, true, kFrameSize, pid + 1, pid + 2);
philipel93e451b2016-10-06 12:25:13 +0200340 ExtractFrame();
341 ExtractFrame();
342 ExtractFrame();
343
344 CheckFrame(0, pid, 0);
345 CheckFrame(1, pid + 2, 0);
346 CheckNoFrame(2);
347}
348
philipelbe7a9e52016-05-19 12:19:35 +0200349TEST_F(TestFrameBuffer2, OneLayerStream) {
350 uint16_t pid = Rand();
351 uint32_t ts = Rand();
352
Sergey Silkin2799e632019-05-17 09:51:39 +0200353 InsertFrame(pid, 0, ts, false, true, kFrameSize);
philipelbe7a9e52016-05-19 12:19:35 +0200354 ExtractFrame();
355 CheckFrame(0, pid, 0);
356 for (int i = 1; i < 10; ++i) {
Sergey Silkin2799e632019-05-17 09:51:39 +0200357 InsertFrame(pid + i, 0, ts + i * kFps10, false, true, kFrameSize,
358 pid + i - 1);
philipelbe7a9e52016-05-19 12:19:35 +0200359 ExtractFrame();
Danil Chapovalov55284022020-02-07 14:53:52 +0100360 time_controller_.AdvanceTime(TimeDelta::Millis(kFps10));
philipelbe7a9e52016-05-19 12:19:35 +0200361 CheckFrame(i, pid + i, 0);
362 }
363}
364
philipelbe7a9e52016-05-19 12:19:35 +0200365TEST_F(TestFrameBuffer2, DropTemporalLayerSlowDecoder) {
366 uint16_t pid = Rand();
367 uint32_t ts = Rand();
368
Sergey Silkin2799e632019-05-17 09:51:39 +0200369 InsertFrame(pid, 0, ts, false, true, kFrameSize);
370 InsertFrame(pid + 1, 0, ts + kFps20, false, true, kFrameSize, pid);
philipelbe7a9e52016-05-19 12:19:35 +0200371 for (int i = 2; i < 10; i += 2) {
372 uint32_t ts_tl0 = ts + i / 2 * kFps10;
Sergey Silkin2799e632019-05-17 09:51:39 +0200373 InsertFrame(pid + i, 0, ts_tl0, false, true, kFrameSize, pid + i - 2);
374 InsertFrame(pid + i + 1, 0, ts_tl0 + kFps20, false, true, kFrameSize,
375 pid + i, pid + i - 1);
philipelbe7a9e52016-05-19 12:19:35 +0200376 }
377
Johannes Kron0c141c52019-08-26 15:04:43 +0200378 EXPECT_CALL(stats_callback_, OnDroppedFrames(1)).Times(3);
379
philipelbe7a9e52016-05-19 12:19:35 +0200380 for (int i = 0; i < 10; ++i) {
381 ExtractFrame();
Danil Chapovalov55284022020-02-07 14:53:52 +0100382 time_controller_.AdvanceTime(TimeDelta::Millis(70));
philipelbe7a9e52016-05-19 12:19:35 +0200383 }
384
385 CheckFrame(0, pid, 0);
386 CheckFrame(1, pid + 1, 0);
387 CheckFrame(2, pid + 2, 0);
388 CheckFrame(3, pid + 4, 0);
389 CheckFrame(4, pid + 6, 0);
390 CheckFrame(5, pid + 8, 0);
391 CheckNoFrame(6);
392 CheckNoFrame(7);
393 CheckNoFrame(8);
394 CheckNoFrame(9);
395}
396
Johannes Kron0c141c52019-08-26 15:04:43 +0200397TEST_F(TestFrameBuffer2, DropFramesIfSystemIsStalled) {
398 uint16_t pid = Rand();
399 uint32_t ts = Rand();
400
401 InsertFrame(pid, 0, ts, false, true, kFrameSize);
402 InsertFrame(pid + 1, 0, ts + 1 * kFps10, false, true, kFrameSize, pid);
403 InsertFrame(pid + 2, 0, ts + 2 * kFps10, false, true, kFrameSize, pid + 1);
404 InsertFrame(pid + 3, 0, ts + 3 * kFps10, false, true, kFrameSize);
405
406 ExtractFrame();
407 // Jump forward in time, simulating the system being stalled for some reason.
Danil Chapovalov55284022020-02-07 14:53:52 +0100408 time_controller_.AdvanceTime(TimeDelta::Millis(3) * kFps10);
Johannes Kron0c141c52019-08-26 15:04:43 +0200409 // Extract one more frame, expect second and third frame to be dropped.
410 EXPECT_CALL(stats_callback_, OnDroppedFrames(2)).Times(1);
411 ExtractFrame();
412
413 CheckFrame(0, pid + 0, 0);
414 CheckFrame(1, pid + 3, 0);
415}
416
417TEST_F(TestFrameBuffer2, DroppedFramesCountedOnClear) {
418 uint16_t pid = Rand();
419 uint32_t ts = Rand();
420
421 InsertFrame(pid, 0, ts, false, true, kFrameSize);
422 for (int i = 1; i < 5; ++i) {
423 InsertFrame(pid + i, 0, ts + i * kFps10, false, true, kFrameSize,
424 pid + i - 1);
425 }
426
427 // All frames should be dropped when Clear is called.
428 EXPECT_CALL(stats_callback_, OnDroppedFrames(5)).Times(1);
429 buffer_->Clear();
430}
431
philipelbe7a9e52016-05-19 12:19:35 +0200432TEST_F(TestFrameBuffer2, InsertLateFrame) {
433 uint16_t pid = Rand();
434 uint32_t ts = Rand();
435
Sergey Silkin2799e632019-05-17 09:51:39 +0200436 InsertFrame(pid, 0, ts, false, true, kFrameSize);
philipelbe7a9e52016-05-19 12:19:35 +0200437 ExtractFrame();
Sergey Silkin2799e632019-05-17 09:51:39 +0200438 InsertFrame(pid + 2, 0, ts, false, true, kFrameSize);
philipelbe7a9e52016-05-19 12:19:35 +0200439 ExtractFrame();
Sergey Silkin2799e632019-05-17 09:51:39 +0200440 InsertFrame(pid + 1, 0, ts, false, true, kFrameSize, pid);
philipelbe7a9e52016-05-19 12:19:35 +0200441 ExtractFrame();
442
443 CheckFrame(0, pid, 0);
444 CheckFrame(1, pid + 2, 0);
445 CheckNoFrame(2);
446}
447
Niels Möller7cca0422019-04-29 16:12:19 +0200448TEST_F(TestFrameBuffer2, ProtectionModeNackFEC) {
philipel4f6cd6a2016-08-03 10:59:32 +0200449 uint16_t pid = Rand();
450 uint32_t ts = Rand();
Niels Möller7cca0422019-04-29 16:12:19 +0200451 constexpr int64_t kRttMs = 200;
452 buffer_->UpdateRtt(kRttMs);
philipel4f6cd6a2016-08-03 10:59:32 +0200453
Niels Möller7cca0422019-04-29 16:12:19 +0200454 // Jitter estimate unaffected by RTT in this protection mode.
Stefan Holmer812ceaf2018-05-15 13:00:10 +0200455 buffer_->SetProtectionMode(kProtectionNackFEC);
Niels Möller7cca0422019-04-29 16:12:19 +0200456 InsertNackedFrame(pid, ts);
457 InsertNackedFrame(pid + 1, ts + 100);
458 InsertNackedFrame(pid + 2, ts + 200);
Sergey Silkin2799e632019-05-17 09:51:39 +0200459 InsertFrame(pid + 3, 0, ts + 300, false, true, kFrameSize);
philipel4f6cd6a2016-08-03 10:59:32 +0200460 ExtractFrame();
Niels Möller7cca0422019-04-29 16:12:19 +0200461 ExtractFrame();
462 ExtractFrame();
463 ExtractFrame();
464 ASSERT_EQ(4u, frames_.size());
465 EXPECT_LT(timing_.GetCurrentJitter(), kRttMs);
466}
467
468TEST_F(TestFrameBuffer2, ProtectionModeNack) {
469 uint16_t pid = Rand();
470 uint32_t ts = Rand();
471 constexpr int64_t kRttMs = 200;
472
473 buffer_->UpdateRtt(kRttMs);
474
475 // Jitter estimate includes RTT (after 3 retransmitted packets)
476 buffer_->SetProtectionMode(kProtectionNack);
477 InsertNackedFrame(pid, ts);
478 InsertNackedFrame(pid + 1, ts + 100);
479 InsertNackedFrame(pid + 2, ts + 200);
Sergey Silkin2799e632019-05-17 09:51:39 +0200480 InsertFrame(pid + 3, 0, ts + 300, false, true, kFrameSize);
Niels Möller7cca0422019-04-29 16:12:19 +0200481 ExtractFrame();
482 ExtractFrame();
483 ExtractFrame();
484 ExtractFrame();
485 ASSERT_EQ(4u, frames_.size());
486
487 EXPECT_GT(timing_.GetCurrentJitter(), kRttMs);
philipel4f6cd6a2016-08-03 10:59:32 +0200488}
489
philipele0b2f152016-09-28 10:23:49 +0200490TEST_F(TestFrameBuffer2, NoContinuousFrame) {
491 uint16_t pid = Rand();
492 uint32_t ts = Rand();
493
Sergey Silkin2799e632019-05-17 09:51:39 +0200494 EXPECT_EQ(-1, InsertFrame(pid + 1, 0, ts, false, true, kFrameSize, pid));
philipele0b2f152016-09-28 10:23:49 +0200495}
496
497TEST_F(TestFrameBuffer2, LastContinuousFrameSingleLayer) {
498 uint16_t pid = Rand();
499 uint32_t ts = Rand();
500
Sergey Silkin2799e632019-05-17 09:51:39 +0200501 EXPECT_EQ(pid, InsertFrame(pid, 0, ts, false, true, kFrameSize));
502 EXPECT_EQ(pid, InsertFrame(pid + 2, 0, ts, false, true, kFrameSize, pid + 1));
503 EXPECT_EQ(pid + 2, InsertFrame(pid + 1, 0, ts, false, true, kFrameSize, pid));
504 EXPECT_EQ(pid + 2,
505 InsertFrame(pid + 4, 0, ts, false, true, kFrameSize, pid + 3));
506 EXPECT_EQ(pid + 5, InsertFrame(pid + 5, 0, ts, false, true, kFrameSize));
philipele0b2f152016-09-28 10:23:49 +0200507}
508
509TEST_F(TestFrameBuffer2, LastContinuousFrameTwoLayers) {
510 uint16_t pid = Rand();
511 uint32_t ts = Rand();
512
Sergey Silkin2799e632019-05-17 09:51:39 +0200513 EXPECT_EQ(pid, InsertFrame(pid, 0, ts, false, false, kFrameSize));
514 EXPECT_EQ(pid, InsertFrame(pid, 1, ts, true, true, kFrameSize));
515 EXPECT_EQ(pid, InsertFrame(pid + 1, 1, ts, true, true, kFrameSize, pid));
516 EXPECT_EQ(pid,
517 InsertFrame(pid + 2, 0, ts, false, false, kFrameSize, pid + 1));
518 EXPECT_EQ(pid, InsertFrame(pid + 2, 1, ts, true, true, kFrameSize, pid + 1));
519 EXPECT_EQ(pid,
520 InsertFrame(pid + 3, 0, ts, false, false, kFrameSize, pid + 2));
521 EXPECT_EQ(pid + 3,
522 InsertFrame(pid + 1, 0, ts, false, false, kFrameSize, pid));
523 EXPECT_EQ(pid + 3,
524 InsertFrame(pid + 3, 1, ts, true, true, kFrameSize, pid + 2));
philipele0b2f152016-09-28 10:23:49 +0200525}
526
philipelfcc60062017-01-18 05:35:20 -0800527TEST_F(TestFrameBuffer2, PictureIdJumpBack) {
528 uint16_t pid = Rand();
529 uint32_t ts = Rand();
530
Sergey Silkin2799e632019-05-17 09:51:39 +0200531 EXPECT_EQ(pid, InsertFrame(pid, 0, ts, false, true, kFrameSize));
532 EXPECT_EQ(pid + 1,
533 InsertFrame(pid + 1, 0, ts + 1, false, true, kFrameSize, pid));
philipelfcc60062017-01-18 05:35:20 -0800534 ExtractFrame();
535 CheckFrame(0, pid, 0);
536
537 // Jump back in pid but increase ts.
Sergey Silkin2799e632019-05-17 09:51:39 +0200538 EXPECT_EQ(pid - 1, InsertFrame(pid - 1, 0, ts + 2, false, true, kFrameSize));
philipelfcc60062017-01-18 05:35:20 -0800539 ExtractFrame();
540 ExtractFrame();
541 CheckFrame(1, pid - 1, 0);
542 CheckNoFrame(2);
543}
544
philipela45102f2017-02-22 05:30:39 -0800545TEST_F(TestFrameBuffer2, StatsCallback) {
546 uint16_t pid = Rand();
547 uint32_t ts = Rand();
548 const int kFrameSize = 5000;
549
ilnik6d5b4d62017-08-30 03:32:14 -0700550 EXPECT_CALL(stats_callback_,
551 OnCompleteFrame(true, kFrameSize, VideoContentType::UNSPECIFIED));
Johannes Kronbfd343b2019-07-01 10:07:50 +0200552 EXPECT_CALL(stats_callback_, OnFrameBufferTimingsUpdated(_, _, _, _, _, _));
philipela45102f2017-02-22 05:30:39 -0800553
554 {
555 std::unique_ptr<FrameObjectFake> frame(new FrameObjectFake());
Niels Möllerb9bfe652019-10-03 08:43:53 +0200556 frame->SetEncodedData(EncodedImageBuffer::Create(kFrameSize));
philipel0fa82a62018-03-19 15:34:53 +0100557 frame->id.picture_id = pid;
558 frame->id.spatial_layer = 0;
Niels Möller23775882018-08-16 10:24:12 +0200559 frame->SetTimestamp(ts);
philipela45102f2017-02-22 05:30:39 -0800560 frame->num_references = 0;
561 frame->inter_layer_predicted = false;
562
Stefan Holmer812ceaf2018-05-15 13:00:10 +0200563 EXPECT_EQ(buffer_->InsertFrame(std::move(frame)), pid);
philipela45102f2017-02-22 05:30:39 -0800564 }
565
566 ExtractFrame();
567 CheckFrame(0, pid, 0);
568}
569
philipel146a48b2017-04-20 04:04:38 -0700570TEST_F(TestFrameBuffer2, ForwardJumps) {
Sergey Silkin2799e632019-05-17 09:51:39 +0200571 EXPECT_EQ(5453, InsertFrame(5453, 0, 1, false, true, kFrameSize));
philipel146a48b2017-04-20 04:04:38 -0700572 ExtractFrame();
Sergey Silkin2799e632019-05-17 09:51:39 +0200573 EXPECT_EQ(5454, InsertFrame(5454, 0, 1, false, true, kFrameSize, 5453));
philipel146a48b2017-04-20 04:04:38 -0700574 ExtractFrame();
Sergey Silkin2799e632019-05-17 09:51:39 +0200575 EXPECT_EQ(15670, InsertFrame(15670, 0, 1, false, true, kFrameSize));
philipel146a48b2017-04-20 04:04:38 -0700576 ExtractFrame();
Sergey Silkin2799e632019-05-17 09:51:39 +0200577 EXPECT_EQ(29804, InsertFrame(29804, 0, 1, false, true, kFrameSize));
philipel146a48b2017-04-20 04:04:38 -0700578 ExtractFrame();
Sergey Silkin2799e632019-05-17 09:51:39 +0200579 EXPECT_EQ(29805, InsertFrame(29805, 0, 1, false, true, kFrameSize, 29804));
philipel146a48b2017-04-20 04:04:38 -0700580 ExtractFrame();
Sergey Silkin2799e632019-05-17 09:51:39 +0200581 EXPECT_EQ(29806, InsertFrame(29806, 0, 1, false, true, kFrameSize, 29805));
philipel146a48b2017-04-20 04:04:38 -0700582 ExtractFrame();
Sergey Silkin2799e632019-05-17 09:51:39 +0200583 EXPECT_EQ(33819, InsertFrame(33819, 0, 1, false, true, kFrameSize));
philipel146a48b2017-04-20 04:04:38 -0700584 ExtractFrame();
Sergey Silkin2799e632019-05-17 09:51:39 +0200585 EXPECT_EQ(41248, InsertFrame(41248, 0, 1, false, true, kFrameSize));
philipel146a48b2017-04-20 04:04:38 -0700586 ExtractFrame();
587}
588
philipelf6842692017-04-28 03:29:15 -0700589TEST_F(TestFrameBuffer2, DuplicateFrames) {
Sergey Silkin2799e632019-05-17 09:51:39 +0200590 EXPECT_EQ(22256, InsertFrame(22256, 0, 1, false, true, kFrameSize));
philipelf6842692017-04-28 03:29:15 -0700591 ExtractFrame();
Sergey Silkin2799e632019-05-17 09:51:39 +0200592 EXPECT_EQ(22256, InsertFrame(22256, 0, 1, false, true, kFrameSize));
philipelf6842692017-04-28 03:29:15 -0700593}
594
philipel112adf92017-06-15 09:06:21 -0700595// TODO(philipel): implement more unittests related to invalid references.
596TEST_F(TestFrameBuffer2, InvalidReferences) {
Sergey Silkin2799e632019-05-17 09:51:39 +0200597 EXPECT_EQ(-1, InsertFrame(0, 0, 1000, false, true, kFrameSize, 2));
598 EXPECT_EQ(1, InsertFrame(1, 0, 2000, false, true, kFrameSize));
philipel112adf92017-06-15 09:06:21 -0700599 ExtractFrame();
Sergey Silkin2799e632019-05-17 09:51:39 +0200600 EXPECT_EQ(2, InsertFrame(2, 0, 3000, false, true, kFrameSize, 1));
philipel112adf92017-06-15 09:06:21 -0700601}
602
philipel3042c2d2017-08-18 04:55:02 -0700603TEST_F(TestFrameBuffer2, KeyframeRequired) {
Sergey Silkin2799e632019-05-17 09:51:39 +0200604 EXPECT_EQ(1, InsertFrame(1, 0, 1000, false, true, kFrameSize));
605 EXPECT_EQ(2, InsertFrame(2, 0, 2000, false, true, kFrameSize, 1));
606 EXPECT_EQ(3, InsertFrame(3, 0, 3000, false, true, kFrameSize));
philipel3042c2d2017-08-18 04:55:02 -0700607 ExtractFrame();
608 ExtractFrame(0, true);
609 ExtractFrame();
610
611 CheckFrame(0, 1, 0);
612 CheckFrame(1, 3, 0);
613 CheckNoFrame(2);
614}
615
philipel9771c502018-03-02 11:06:27 +0100616TEST_F(TestFrameBuffer2, KeyframeClearsFullBuffer) {
617 const int kMaxBufferSize = 600;
618
619 for (int i = 1; i <= kMaxBufferSize; ++i)
Sergey Silkin2799e632019-05-17 09:51:39 +0200620 EXPECT_EQ(-1, InsertFrame(i, 0, i * 1000, false, true, kFrameSize, i - 1));
philipel9771c502018-03-02 11:06:27 +0100621 ExtractFrame();
622 CheckNoFrame(0);
623
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100624 EXPECT_EQ(kMaxBufferSize + 1,
625 InsertFrame(kMaxBufferSize + 1, 0, (kMaxBufferSize + 1) * 1000,
Sergey Silkin2799e632019-05-17 09:51:39 +0200626 false, true, kFrameSize));
philipel9771c502018-03-02 11:06:27 +0100627 ExtractFrame();
628 CheckFrame(1, kMaxBufferSize + 1, 0);
629}
630
philipel798b2822018-06-11 13:10:14 +0200631TEST_F(TestFrameBuffer2, DontUpdateOnUndecodableFrame) {
Sergey Silkin2799e632019-05-17 09:51:39 +0200632 InsertFrame(1, 0, 0, false, true, kFrameSize);
philipel798b2822018-06-11 13:10:14 +0200633 ExtractFrame(0, true);
Sergey Silkin2799e632019-05-17 09:51:39 +0200634 InsertFrame(3, 0, 0, false, true, kFrameSize, 2, 0);
635 InsertFrame(3, 0, 0, false, true, kFrameSize, 0);
636 InsertFrame(2, 0, 0, false, true, kFrameSize);
philipel798b2822018-06-11 13:10:14 +0200637 ExtractFrame(0, true);
638 ExtractFrame(0, true);
639}
640
philipel6d216502018-10-22 14:36:45 +0200641TEST_F(TestFrameBuffer2, DontDecodeOlderTimestamp) {
Sergey Silkin2799e632019-05-17 09:51:39 +0200642 InsertFrame(2, 0, 1, false, true, kFrameSize);
643 InsertFrame(1, 0, 2, false, true,
644 kFrameSize); // Older picture id but newer timestamp.
philipel6d216502018-10-22 14:36:45 +0200645 ExtractFrame(0);
646 ExtractFrame(0);
647 CheckFrame(0, 1, 0);
648 CheckNoFrame(1);
649
Sergey Silkin2799e632019-05-17 09:51:39 +0200650 InsertFrame(3, 0, 4, false, true, kFrameSize);
651 InsertFrame(4, 0, 3, false, true,
652 kFrameSize); // Newer picture id but older timestamp.
philipel6d216502018-10-22 14:36:45 +0200653 ExtractFrame(0);
654 ExtractFrame(0);
655 CheckFrame(2, 3, 0);
656 CheckNoFrame(3);
657}
658
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100659TEST_F(TestFrameBuffer2, CombineFramesToSuperframe) {
660 uint16_t pid = Rand();
661 uint32_t ts = Rand();
662
Sergey Silkin2799e632019-05-17 09:51:39 +0200663 InsertFrame(pid, 0, ts, false, false, kFrameSize);
664 InsertFrame(pid, 1, ts, true, true, 2 * kFrameSize);
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100665 ExtractFrame(0);
666 ExtractFrame(0);
Sergey Silkin61832dd2018-12-20 14:32:14 +0100667 CheckFrame(0, pid, 1);
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100668 CheckNoFrame(1);
669 // Two frames should be combined and returned together.
Sergey Silkin2799e632019-05-17 09:51:39 +0200670 CheckFrameSize(0, 3 * kFrameSize);
671
672 EXPECT_EQ(frames_[0]->SpatialIndex(), 1);
673 EXPECT_EQ(frames_[0]->SpatialLayerFrameSize(0), kFrameSize);
674 EXPECT_EQ(frames_[0]->SpatialLayerFrameSize(1), 2 * kFrameSize);
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100675}
676
677TEST_F(TestFrameBuffer2, HigherSpatialLayerNonDecodable) {
678 uint16_t pid = Rand();
679 uint32_t ts = Rand();
680
Sergey Silkin2799e632019-05-17 09:51:39 +0200681 InsertFrame(pid, 0, ts, false, false, kFrameSize);
682 InsertFrame(pid, 1, ts, true, true, kFrameSize);
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100683
684 ExtractFrame(0);
Sergey Silkin61832dd2018-12-20 14:32:14 +0100685 CheckFrame(0, pid, 1);
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100686
Sergey Silkin2799e632019-05-17 09:51:39 +0200687 InsertFrame(pid + 1, 1, ts + kFps20, false, true, kFrameSize, pid);
688 InsertFrame(pid + 2, 0, ts + kFps10, false, false, kFrameSize, pid);
689 InsertFrame(pid + 2, 1, ts + kFps10, true, true, kFrameSize, pid + 1);
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100690
Danil Chapovalov55284022020-02-07 14:53:52 +0100691 time_controller_.AdvanceTime(TimeDelta::Millis(1000));
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100692 // Frame pid+1 is decodable but too late.
693 // In superframe pid+2 frame sid=0 is decodable, but frame sid=1 is not.
694 // Incorrect implementation might skip pid+1 frame and output undecodable
695 // pid+2 instead.
696 ExtractFrame();
697 ExtractFrame();
698 CheckFrame(1, pid + 1, 1);
Sergey Silkin61832dd2018-12-20 14:32:14 +0100699 CheckFrame(2, pid + 2, 1);
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100700}
701
philipelbe7a9e52016-05-19 12:19:35 +0200702} // namespace video_coding
703} // namespace webrtc