blob: c889769159ff98915177437db3baf1f36ad9feed [file] [log] [blame]
ilnik04f4d122017-06-19 07:18:55 -07001/*
2 * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include <vector>
12
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020013#include "modules/video_coding/encoded_frame.h"
14#include "modules/video_coding/generic_encoder.h"
15#include "modules/video_coding/include/video_coding_defines.h"
Ilya Nikolaevskiy764aeb72018-04-03 10:01:52 +020016#include "rtc_base/fakeclock.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020017#include "test/gtest.h"
ilnik04f4d122017-06-19 07:18:55 -070018
19namespace webrtc {
20namespace test {
21namespace {
22inline size_t FrameSize(const size_t& min_frame_size,
23 const size_t& max_frame_size,
24 const int& s,
25 const int& i) {
26 return min_frame_size + (s + 1) * i % (max_frame_size - min_frame_size);
27}
28
29class FakeEncodedImageCallback : public EncodedImageCallback {
30 public:
Ilya Nikolaevskiyd79314f2017-10-23 10:45:37 +020031 FakeEncodedImageCallback()
Ilya Nikolaevskiy76f2a852017-11-16 14:33:53 +010032 : last_frame_was_timing_(false),
33 num_frames_dropped_(0),
34 last_capture_timestamp_(-1) {}
ilnik04f4d122017-06-19 07:18:55 -070035 Result OnEncodedImage(const EncodedImage& encoded_image,
36 const CodecSpecificInfo* codec_specific_info,
37 const RTPFragmentationHeader* fragmentation) override {
sprangba050a62017-08-18 02:51:12 -070038 last_frame_was_timing_ =
Ilya Nikolaevskiyb6c462d2018-06-05 15:21:32 +020039 encoded_image.timing_.flags != VideoSendTiming::kInvalid &&
40 encoded_image.timing_.flags != VideoSendTiming::kNotTriggered;
Ilya Nikolaevskiy76f2a852017-11-16 14:33:53 +010041 last_capture_timestamp_ = encoded_image.capture_time_ms_;
mflodman351424e2017-08-10 02:43:14 -070042 return Result(Result::OK);
ilnik04f4d122017-06-19 07:18:55 -070043 };
44
Ilya Nikolaevskiyd79314f2017-10-23 10:45:37 +020045 void OnDroppedFrame(DropReason reason) override { ++num_frames_dropped_; }
46
ilnik04f4d122017-06-19 07:18:55 -070047 bool WasTimingFrame() { return last_frame_was_timing_; }
48
Ilya Nikolaevskiyd79314f2017-10-23 10:45:37 +020049 size_t GetNumFramesDropped() { return num_frames_dropped_; }
50
Ilya Nikolaevskiy76f2a852017-11-16 14:33:53 +010051 int64_t GetLastCaptureTimestamp() { return last_capture_timestamp_; }
52
ilnik04f4d122017-06-19 07:18:55 -070053 private:
54 bool last_frame_was_timing_;
Ilya Nikolaevskiyd79314f2017-10-23 10:45:37 +020055 size_t num_frames_dropped_;
Ilya Nikolaevskiy76f2a852017-11-16 14:33:53 +010056 int64_t last_capture_timestamp_;
ilnik04f4d122017-06-19 07:18:55 -070057};
58
59enum class FrameType {
60 kNormal,
61 kTiming,
62 kDropped,
63};
64
65// Emulates |num_frames| on |num_streams| frames with capture timestamps
66// increased by 1 from 0. Size of each frame is between
67// |min_frame_size| and |max_frame_size|, outliers are counted relatevely to
68// |average_frame_sizes[]| for each stream.
69std::vector<std::vector<FrameType>> GetTimingFrames(
70 const int64_t delay_ms,
71 const size_t min_frame_size,
72 const size_t max_frame_size,
73 std::vector<size_t> average_frame_sizes,
74 const int num_streams,
75 const int num_frames) {
76 FakeEncodedImageCallback sink;
77 VCMEncodedFrameCallback callback(&sink, nullptr);
78 const size_t kFramerate = 30;
79 callback.SetTimingFramesThresholds(
80 {delay_ms, kDefaultOutlierFrameSizePercent});
81 callback.OnFrameRateChanged(kFramerate);
82 int s, i;
83 std::vector<std::vector<FrameType>> result(num_streams);
84 for (s = 0; s < num_streams; ++s)
85 callback.OnTargetBitrateChanged(average_frame_sizes[s] * kFramerate, s);
86 int64_t current_timestamp = 0;
87 for (i = 0; i < num_frames; ++i) {
88 current_timestamp += 1;
89 for (s = 0; s < num_streams; ++s) {
90 // every (5+s)-th frame is dropped on s-th stream by design.
91 bool dropped = i % (5 + s) == 0;
92
93 EncodedImage image;
94 CodecSpecificInfo codec_specific;
95 image._length = FrameSize(min_frame_size, max_frame_size, s, i);
96 image.capture_time_ms_ = current_timestamp;
Niels Möller23775882018-08-16 10:24:12 +020097 image.SetTimestamp(static_cast<uint32_t>(current_timestamp * 90));
ilnik04f4d122017-06-19 07:18:55 -070098 codec_specific.codecType = kVideoCodecGeneric;
99 codec_specific.codecSpecific.generic.simulcast_idx = s;
Ilya Nikolaevskiy76f2a852017-11-16 14:33:53 +0100100 callback.OnEncodeStarted(static_cast<uint32_t>(current_timestamp * 90),
101 current_timestamp, s);
ilnik04f4d122017-06-19 07:18:55 -0700102 if (dropped) {
103 result[s].push_back(FrameType::kDropped);
104 continue;
105 }
106 callback.OnEncodedImage(image, &codec_specific, nullptr);
107 if (sink.WasTimingFrame()) {
108 result[s].push_back(FrameType::kTiming);
109 } else {
110 result[s].push_back(FrameType::kNormal);
111 }
112 }
113 }
114 return result;
115}
116} // namespace
117
118TEST(TestVCMEncodedFrameCallback, MarksTimingFramesPeriodicallyTogether) {
119 const int64_t kDelayMs = 29;
120 const size_t kMinFrameSize = 10;
121 const size_t kMaxFrameSize = 20;
122 const int kNumFrames = 1000;
123 const int kNumStreams = 3;
124 // No outliers as 1000 is larger than anything from range [10,20].
125 const std::vector<size_t> kAverageSize = {1000, 1000, 1000};
126 auto frames = GetTimingFrames(kDelayMs, kMinFrameSize, kMaxFrameSize,
127 kAverageSize, kNumStreams, kNumFrames);
128 // Timing frames should be tirggered every delayMs.
129 // As no outliers are expected, frames on all streams have to be
130 // marked together.
131 int last_timing_frame = -1;
132 for (int i = 0; i < kNumFrames; ++i) {
133 int num_normal = 0;
134 int num_timing = 0;
135 int num_dropped = 0;
136 for (int s = 0; s < kNumStreams; ++s) {
137 if (frames[s][i] == FrameType::kTiming) {
138 ++num_timing;
139 } else if (frames[s][i] == FrameType::kNormal) {
140 ++num_normal;
141 } else {
142 ++num_dropped;
143 }
144 }
145 // Can't have both normal and timing frames at the same timstamp.
146 EXPECT_TRUE(num_timing == 0 || num_normal == 0);
147 if (num_dropped < kNumStreams) {
148 if (last_timing_frame == -1 || i >= last_timing_frame + kDelayMs) {
149 // If didn't have timing frames for a period, current sent frame has to
150 // be one. No normal frames should be sent.
151 EXPECT_EQ(num_normal, 0);
152 } else {
153 // No unneeded timing frames should be sent.
154 EXPECT_EQ(num_timing, 0);
155 }
156 }
157 if (num_timing > 0)
158 last_timing_frame = i;
159 }
160}
161
162TEST(TestVCMEncodedFrameCallback, MarksOutliers) {
163 const int64_t kDelayMs = 29;
164 const size_t kMinFrameSize = 2495;
165 const size_t kMaxFrameSize = 2505;
166 const int kNumFrames = 1000;
167 const int kNumStreams = 3;
168 // Possible outliers as 1000 lies in range [995, 1005].
169 const std::vector<size_t> kAverageSize = {998, 1000, 1004};
170 auto frames = GetTimingFrames(kDelayMs, kMinFrameSize, kMaxFrameSize,
171 kAverageSize, kNumStreams, kNumFrames);
172 // All outliers should be marked.
173 for (int i = 0; i < kNumFrames; ++i) {
174 for (int s = 0; s < kNumStreams; ++s) {
175 if (FrameSize(kMinFrameSize, kMaxFrameSize, s, i) >=
176 kAverageSize[s] * kDefaultOutlierFrameSizePercent / 100) {
177 // Too big frame. May be dropped or timing, but not normal.
178 EXPECT_NE(frames[s][i], FrameType::kNormal);
179 }
180 }
181 }
182}
183
sprangba050a62017-08-18 02:51:12 -0700184TEST(TestVCMEncodedFrameCallback, NoTimingFrameIfNoEncodeStartTime) {
185 EncodedImage image;
186 CodecSpecificInfo codec_specific;
187 int64_t timestamp = 1;
188 image._length = 500;
189 image.capture_time_ms_ = timestamp;
Niels Möller23775882018-08-16 10:24:12 +0200190 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
sprangba050a62017-08-18 02:51:12 -0700191 codec_specific.codecType = kVideoCodecGeneric;
192 codec_specific.codecSpecific.generic.simulcast_idx = 0;
193 FakeEncodedImageCallback sink;
194 VCMEncodedFrameCallback callback(&sink, nullptr);
195 VideoCodec::TimingFrameTriggerThresholds thresholds;
196 thresholds.delay_ms = 1; // Make all frames timing frames.
197 callback.SetTimingFramesThresholds(thresholds);
Ilya Nikolaevskiye0da9ea2017-11-08 14:39:02 +0100198 callback.OnTargetBitrateChanged(500, 0);
sprangba050a62017-08-18 02:51:12 -0700199
200 // Verify a single frame works with encode start time set.
Ilya Nikolaevskiy76f2a852017-11-16 14:33:53 +0100201 callback.OnEncodeStarted(static_cast<uint32_t>(timestamp * 90), timestamp, 0);
sprangba050a62017-08-18 02:51:12 -0700202 callback.OnEncodedImage(image, &codec_specific, nullptr);
203 EXPECT_TRUE(sink.WasTimingFrame());
204
205 // New frame, now skip OnEncodeStarted. Should not result in timing frame.
206 image.capture_time_ms_ = ++timestamp;
Niels Möller23775882018-08-16 10:24:12 +0200207 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
sprangba050a62017-08-18 02:51:12 -0700208 callback.OnEncodedImage(image, &codec_specific, nullptr);
209 EXPECT_FALSE(sink.WasTimingFrame());
210}
211
Ilya Nikolaevskiy764aeb72018-04-03 10:01:52 +0200212TEST(TestVCMEncodedFrameCallback, AdjustsCaptureTimeForInternalSourceEncoder) {
213 rtc::ScopedFakeClock clock;
214 clock.SetTimeMicros(1234567);
215 EncodedImage image;
216 CodecSpecificInfo codec_specific;
217 const int64_t kEncodeStartDelayMs = 2;
218 const int64_t kEncodeFinishDelayMs = 10;
219 int64_t timestamp = 1;
220 image._length = 500;
221 image.capture_time_ms_ = timestamp;
Niels Möller23775882018-08-16 10:24:12 +0200222 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
Ilya Nikolaevskiy764aeb72018-04-03 10:01:52 +0200223 codec_specific.codecType = kVideoCodecGeneric;
224 codec_specific.codecSpecific.generic.simulcast_idx = 0;
225 FakeEncodedImageCallback sink;
226 VCMEncodedFrameCallback callback(&sink, nullptr);
227 callback.SetInternalSource(true);
228 VideoCodec::TimingFrameTriggerThresholds thresholds;
229 thresholds.delay_ms = 1; // Make all frames timing frames.
230 callback.SetTimingFramesThresholds(thresholds);
231 callback.OnTargetBitrateChanged(500, 0);
232
233 // Verify a single frame without encode timestamps isn't a timing frame.
234 callback.OnEncodedImage(image, &codec_specific, nullptr);
235 EXPECT_FALSE(sink.WasTimingFrame());
236
237 // New frame, but this time with encode timestamps set in timing_.
238 // This should be a timing frame.
239 image.capture_time_ms_ = ++timestamp;
Niels Möller23775882018-08-16 10:24:12 +0200240 image.SetTimestamp(static_cast<uint32_t>(timestamp * 90));
Ilya Nikolaevskiy764aeb72018-04-03 10:01:52 +0200241 image.timing_.encode_start_ms = timestamp + kEncodeStartDelayMs;
242 image.timing_.encode_finish_ms = timestamp + kEncodeFinishDelayMs;
243 callback.OnEncodedImage(image, &codec_specific, nullptr);
244 EXPECT_TRUE(sink.WasTimingFrame());
245 // Frame is captured kEncodeFinishDelayMs before it's encoded, so restored
246 // capture timestamp should be kEncodeFinishDelayMs in the past.
247 EXPECT_EQ(
248 sink.GetLastCaptureTimestamp(),
249 clock.TimeNanos() / rtc::kNumNanosecsPerMillisec - kEncodeFinishDelayMs);
250}
251
Ilya Nikolaevskiyd79314f2017-10-23 10:45:37 +0200252TEST(TestVCMEncodedFrameCallback, NotifiesAboutDroppedFrames) {
253 EncodedImage image;
254 CodecSpecificInfo codec_specific;
Ilya Nikolaevskiy76f2a852017-11-16 14:33:53 +0100255 const int64_t kTimestampMs1 = 47721840;
256 const int64_t kTimestampMs2 = 47721850;
257 const int64_t kTimestampMs3 = 47721860;
258 const int64_t kTimestampMs4 = 47721870;
Ilya Nikolaevskiyd79314f2017-10-23 10:45:37 +0200259 codec_specific.codecType = kVideoCodecGeneric;
260 codec_specific.codecSpecific.generic.simulcast_idx = 0;
261 FakeEncodedImageCallback sink;
262 VCMEncodedFrameCallback callback(&sink, nullptr);
Ilya Nikolaevskiye0da9ea2017-11-08 14:39:02 +0100263 // Any non-zero bitrate needed to be set before the first frame.
264 callback.OnTargetBitrateChanged(500, 0);
Ilya Nikolaevskiy76f2a852017-11-16 14:33:53 +0100265 image.capture_time_ms_ = kTimestampMs1;
Niels Möller23775882018-08-16 10:24:12 +0200266 image.SetTimestamp(static_cast<uint32_t>(image.capture_time_ms_ * 90));
267 callback.OnEncodeStarted(image.Timestamp(), image.capture_time_ms_, 0);
Ilya Nikolaevskiyd79314f2017-10-23 10:45:37 +0200268 EXPECT_EQ(0u, sink.GetNumFramesDropped());
Ilya Nikolaevskiyd79314f2017-10-23 10:45:37 +0200269 callback.OnEncodedImage(image, &codec_specific, nullptr);
Ilya Nikolaevskiy76f2a852017-11-16 14:33:53 +0100270
271 image.capture_time_ms_ = kTimestampMs2;
Niels Möller23775882018-08-16 10:24:12 +0200272 image.SetTimestamp(static_cast<uint32_t>(image.capture_time_ms_ * 90));
273 callback.OnEncodeStarted(image.Timestamp(), image.capture_time_ms_, 0);
Ilya Nikolaevskiyd79314f2017-10-23 10:45:37 +0200274 // No OnEncodedImageCall for timestamp2. Yet, at this moment it's not known
275 // that frame with timestamp2 was dropped.
276 EXPECT_EQ(0u, sink.GetNumFramesDropped());
Ilya Nikolaevskiy76f2a852017-11-16 14:33:53 +0100277
278 image.capture_time_ms_ = kTimestampMs3;
Niels Möller23775882018-08-16 10:24:12 +0200279 image.SetTimestamp(static_cast<uint32_t>(image.capture_time_ms_ * 90));
280 callback.OnEncodeStarted(image.Timestamp(), image.capture_time_ms_, 0);
Ilya Nikolaevskiyd79314f2017-10-23 10:45:37 +0200281 callback.OnEncodedImage(image, &codec_specific, nullptr);
282 EXPECT_EQ(1u, sink.GetNumFramesDropped());
Ilya Nikolaevskiy76f2a852017-11-16 14:33:53 +0100283
284 image.capture_time_ms_ = kTimestampMs4;
Niels Möller23775882018-08-16 10:24:12 +0200285 image.SetTimestamp(static_cast<uint32_t>(image.capture_time_ms_ * 90));
286 callback.OnEncodeStarted(image.Timestamp(), image.capture_time_ms_, 0);
Ilya Nikolaevskiyd79314f2017-10-23 10:45:37 +0200287 callback.OnEncodedImage(image, &codec_specific, nullptr);
288 EXPECT_EQ(1u, sink.GetNumFramesDropped());
289}
290
Ilya Nikolaevskiy76f2a852017-11-16 14:33:53 +0100291TEST(TestVCMEncodedFrameCallback, RestoresCaptureTimestamps) {
292 EncodedImage image;
293 CodecSpecificInfo codec_specific;
294 const int64_t kTimestampMs = 123456;
295 codec_specific.codecType = kVideoCodecGeneric;
296 codec_specific.codecSpecific.generic.simulcast_idx = 0;
297 FakeEncodedImageCallback sink;
298 VCMEncodedFrameCallback callback(&sink, nullptr);
299 // Any non-zero bitrate needed to be set before the first frame.
300 callback.OnTargetBitrateChanged(500, 0);
301 image.capture_time_ms_ = kTimestampMs; // Incorrect timesetamp.
Niels Möller23775882018-08-16 10:24:12 +0200302 image.SetTimestamp(static_cast<uint32_t>(image.capture_time_ms_ * 90));
303 callback.OnEncodeStarted(image.Timestamp(), image.capture_time_ms_, 0);
Ilya Nikolaevskiy76f2a852017-11-16 14:33:53 +0100304 image.capture_time_ms_ = 0; // Incorrect timesetamp.
305 callback.OnEncodedImage(image, &codec_specific, nullptr);
306 EXPECT_EQ(kTimestampMs, sink.GetLastCaptureTimestamp());
307}
308
ilnik04f4d122017-06-19 07:18:55 -0700309} // namespace test
310} // namespace webrtc