blob: 7cc3a86e5c589899e56dfd5bf0ccb115c205c847 [file] [log] [blame]
Sebastian Janssond4c5d632018-07-10 12:57:37 +02001/*
2 * Copyright 2018 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10#ifndef VIDEO_VIDEO_ANALYZER_H_
11#define VIDEO_VIDEO_ANALYZER_H_
12
13#include <deque>
14#include <map>
15#include <memory>
16#include <string>
17#include <vector>
18
Niels Möller1c931c42018-12-18 16:08:11 +010019#include "api/video/video_source_interface.h"
Yves Gerey79e9f4b2019-04-13 18:59:53 +020020#include "rtc_base/numerics/running_statistics.h"
Bjorn Terelius5c2f1f02019-01-16 17:45:05 +010021#include "rtc_base/time_utils.h"
Sebastian Janssond4c5d632018-07-10 12:57:37 +020022#include "test/layer_filtering_transport.h"
23#include "test/rtp_file_writer.h"
Sebastian Janssond4c5d632018-07-10 12:57:37 +020024
25namespace webrtc {
26
27class VideoAnalyzer : public PacketReceiver,
28 public Transport,
Niels Möller88be9722018-10-10 10:58:52 +020029 public rtc::VideoSinkInterface<VideoFrame> {
Sebastian Janssond4c5d632018-07-10 12:57:37 +020030 public:
Yves Gerey79e9f4b2019-04-13 18:59:53 +020031 using Statistics = RunningStatistics<double>;
32
Sebastian Janssond4c5d632018-07-10 12:57:37 +020033 VideoAnalyzer(test::LayerFilteringTransport* transport,
34 const std::string& test_label,
35 double avg_psnr_threshold,
36 double avg_ssim_threshold,
37 int duration_frames,
38 FILE* graph_data_output_file,
39 const std::string& graph_title,
40 uint32_t ssrc_to_analyze,
41 uint32_t rtx_ssrc_to_analyze,
42 size_t selected_stream,
43 int selected_sl,
44 int selected_tl,
45 bool is_quick_test_enabled,
46 Clock* clock,
Artem Titovff7730d2019-04-02 13:46:53 +020047 std::string rtp_dump_name,
48 test::SingleThreadedTaskQueueForTesting* task_queue);
Sebastian Janssond4c5d632018-07-10 12:57:37 +020049 ~VideoAnalyzer();
50
51 virtual void SetReceiver(PacketReceiver* receiver);
Niels Möller1c931c42018-12-18 16:08:11 +010052 void SetSource(rtc::VideoSourceInterface<VideoFrame>* video_source,
Sebastian Janssonf1f363f2018-08-13 14:24:58 +020053 bool respect_sink_wants);
Sebastian Janssond4c5d632018-07-10 12:57:37 +020054 void SetCall(Call* call);
55 void SetSendStream(VideoSendStream* stream);
56 void SetReceiveStream(VideoReceiveStream* stream);
Christoffer Rodbroc2a02882018-08-07 14:10:56 +020057 void SetAudioReceiveStream(AudioReceiveStream* recv_stream);
58
Sebastian Janssond4c5d632018-07-10 12:57:37 +020059 rtc::VideoSinkInterface<VideoFrame>* InputInterface();
60 rtc::VideoSourceInterface<VideoFrame>* OutputInterface();
61
62 DeliveryStatus DeliverPacket(MediaType media_type,
63 rtc::CopyOnWriteBuffer packet,
Niels Möller70082872018-08-07 11:03:12 +020064 int64_t packet_time_us) override;
Sebastian Janssond4c5d632018-07-10 12:57:37 +020065
66 void PreEncodeOnFrame(const VideoFrame& video_frame);
Niels Möller88be9722018-10-10 10:58:52 +020067 void PostEncodeOnFrame(size_t stream_id, uint32_t timestamp);
Sebastian Janssond4c5d632018-07-10 12:57:37 +020068
69 bool SendRtp(const uint8_t* packet,
70 size_t length,
71 const PacketOptions& options) override;
72
73 bool SendRtcp(const uint8_t* packet, size_t length) override;
74 void OnFrame(const VideoFrame& video_frame) override;
75 void Wait();
76
Sebastian Janssond4c5d632018-07-10 12:57:37 +020077 void StartMeasuringCpuProcessTime();
78 void StopMeasuringCpuProcessTime();
79 void StartExcludingCpuThreadTime();
80 void StopExcludingCpuThreadTime();
81 double GetCpuUsagePercent();
82
83 test::LayerFilteringTransport* const transport_;
84 PacketReceiver* receiver_;
85
86 private:
87 struct FrameComparison {
88 FrameComparison();
89 FrameComparison(const VideoFrame& reference,
90 const VideoFrame& render,
91 bool dropped,
92 int64_t input_time_ms,
93 int64_t send_time_ms,
94 int64_t recv_time_ms,
95 int64_t render_time_ms,
96 size_t encoded_frame_size);
97 FrameComparison(bool dropped,
98 int64_t input_time_ms,
99 int64_t send_time_ms,
100 int64_t recv_time_ms,
101 int64_t render_time_ms,
102 size_t encoded_frame_size);
103
104 absl::optional<VideoFrame> reference;
105 absl::optional<VideoFrame> render;
106 bool dropped;
107 int64_t input_time_ms;
108 int64_t send_time_ms;
109 int64_t recv_time_ms;
110 int64_t render_time_ms;
111 size_t encoded_frame_size;
112 };
113
114 struct Sample {
115 Sample(int dropped,
116 int64_t input_time_ms,
117 int64_t send_time_ms,
118 int64_t recv_time_ms,
119 int64_t render_time_ms,
120 size_t encoded_frame_size,
121 double psnr,
122 double ssim);
123
124 int dropped;
125 int64_t input_time_ms;
126 int64_t send_time_ms;
127 int64_t recv_time_ms;
128 int64_t render_time_ms;
129 size_t encoded_frame_size;
130 double psnr;
131 double ssim;
132 };
133
Sebastian Janssond4c5d632018-07-10 12:57:37 +0200134 // Implements VideoSinkInterface to receive captured frames from a
135 // FrameGeneratorCapturer. Implements VideoSourceInterface to be able to act
136 // as a source to VideoSendStream.
137 // It forwards all input frames to the VideoAnalyzer for later comparison and
138 // forwards the captured frames to the VideoSendStream.
139 class CapturedFrameForwarder : public rtc::VideoSinkInterface<VideoFrame>,
140 public rtc::VideoSourceInterface<VideoFrame> {
141 public:
Ilya Nikolaevskiy6957abe2019-01-29 16:33:04 +0100142 CapturedFrameForwarder(VideoAnalyzer* analyzer,
143 Clock* clock,
Ilya Nikolaevskiy85fc3252019-02-11 10:41:50 +0100144 int frames_to_process);
Niels Möller1c931c42018-12-18 16:08:11 +0100145 void SetSource(rtc::VideoSourceInterface<VideoFrame>* video_source);
Sebastian Janssond4c5d632018-07-10 12:57:37 +0200146
147 private:
148 void OnFrame(const VideoFrame& video_frame) override;
149
150 // Called when |send_stream_.SetSource()| is called.
151 void AddOrUpdateSink(rtc::VideoSinkInterface<VideoFrame>* sink,
152 const rtc::VideoSinkWants& wants) override;
153
154 // Called by |send_stream_| when |send_stream_.SetSource()| is called.
155 void RemoveSink(rtc::VideoSinkInterface<VideoFrame>* sink) override;
156
157 VideoAnalyzer* const analyzer_;
158 rtc::CriticalSection crit_;
159 rtc::VideoSinkInterface<VideoFrame>* send_stream_input_
160 RTC_GUARDED_BY(crit_);
Niels Möller1c931c42018-12-18 16:08:11 +0100161 VideoSourceInterface<VideoFrame>* video_source_;
Sebastian Janssond4c5d632018-07-10 12:57:37 +0200162 Clock* clock_;
Ilya Nikolaevskiy6957abe2019-01-29 16:33:04 +0100163 int captured_frames_ RTC_GUARDED_BY(crit_);
164 int frames_to_process_ RTC_GUARDED_BY(crit_);
Sebastian Janssond4c5d632018-07-10 12:57:37 +0200165 };
166
167 struct FrameWithPsnr {
168 double psnr;
169 VideoFrame frame;
170 };
171
172 bool IsInSelectedSpatialAndTemporalLayer(const uint8_t* packet,
173 size_t length,
174 const RTPHeader& header);
175
176 void AddFrameComparison(const VideoFrame& reference,
177 const VideoFrame& render,
178 bool dropped,
179 int64_t render_time_ms)
180 RTC_EXCLUSIVE_LOCKS_REQUIRED(crit_);
181
Sebastian Janssond4c5d632018-07-10 12:57:37 +0200182 void PollStats();
Niels Möller4731f002019-05-03 09:34:24 +0200183 static void FrameComparisonThread(void* obj);
Sebastian Janssond4c5d632018-07-10 12:57:37 +0200184 bool CompareFrames();
185 bool PopComparison(FrameComparison* comparison);
186 // Increment counter for number of frames received for comparison.
187 void FrameRecorded();
188 // Returns true if all frames to be compared have been taken from the queue.
189 bool AllFramesRecorded();
190 // Increase count of number of frames processed. Returns true if this was the
191 // last frame to be processed.
192 bool FrameProcessed();
193 void PrintResults();
194 void PerformFrameComparison(const FrameComparison& comparison);
Yves Gerey79e9f4b2019-04-13 18:59:53 +0200195 void PrintResult(const char* result_type, Statistics stats, const char* unit);
Johannes Krona1b99b32019-07-30 15:08:16 +0200196 void PrintResultWithExternalMean(const char* result_type,
197 double mean,
198 Statistics stats,
199 const char* unit);
Sebastian Janssond4c5d632018-07-10 12:57:37 +0200200 void PrintSamplesToFile(void);
201 double GetAverageMediaBitrateBps();
202 void AddCapturedFrameForComparison(const VideoFrame& video_frame);
203
204 Call* call_;
205 VideoSendStream* send_stream_;
206 VideoReceiveStream* receive_stream_;
Christoffer Rodbroc2a02882018-08-07 14:10:56 +0200207 AudioReceiveStream* audio_receive_stream_;
Sebastian Janssond4c5d632018-07-10 12:57:37 +0200208 CapturedFrameForwarder captured_frame_forwarder_;
209 const std::string test_label_;
210 FILE* const graph_data_output_file_;
211 const std::string graph_title_;
212 const uint32_t ssrc_to_analyze_;
213 const uint32_t rtx_ssrc_to_analyze_;
214 const size_t selected_stream_;
215 const int selected_sl_;
216 const int selected_tl_;
Sebastian Janssond4c5d632018-07-10 12:57:37 +0200217
218 rtc::CriticalSection comparison_lock_;
219 std::vector<Sample> samples_ RTC_GUARDED_BY(comparison_lock_);
Yves Gerey79e9f4b2019-04-13 18:59:53 +0200220 Statistics sender_time_ RTC_GUARDED_BY(comparison_lock_);
221 Statistics receiver_time_ RTC_GUARDED_BY(comparison_lock_);
222 Statistics network_time_ RTC_GUARDED_BY(comparison_lock_);
223 Statistics psnr_ RTC_GUARDED_BY(comparison_lock_);
224 Statistics ssim_ RTC_GUARDED_BY(comparison_lock_);
225 Statistics end_to_end_ RTC_GUARDED_BY(comparison_lock_);
226 Statistics rendered_delta_ RTC_GUARDED_BY(comparison_lock_);
227 Statistics encoded_frame_size_ RTC_GUARDED_BY(comparison_lock_);
228 Statistics encode_frame_rate_ RTC_GUARDED_BY(comparison_lock_);
229 Statistics encode_time_ms_ RTC_GUARDED_BY(comparison_lock_);
230 Statistics encode_usage_percent_ RTC_GUARDED_BY(comparison_lock_);
Johannes Krona1b99b32019-07-30 15:08:16 +0200231 double mean_decode_time_ms_ RTC_GUARDED_BY(comparison_lock_);
Yves Gerey79e9f4b2019-04-13 18:59:53 +0200232 Statistics decode_time_ms_ RTC_GUARDED_BY(comparison_lock_);
233 Statistics decode_time_max_ms_ RTC_GUARDED_BY(comparison_lock_);
234 Statistics media_bitrate_bps_ RTC_GUARDED_BY(comparison_lock_);
235 Statistics fec_bitrate_bps_ RTC_GUARDED_BY(comparison_lock_);
236 Statistics send_bandwidth_bps_ RTC_GUARDED_BY(comparison_lock_);
237 Statistics memory_usage_ RTC_GUARDED_BY(comparison_lock_);
Yves Gerey79e9f4b2019-04-13 18:59:53 +0200238 Statistics audio_expand_rate_ RTC_GUARDED_BY(comparison_lock_);
239 Statistics audio_accelerate_rate_ RTC_GUARDED_BY(comparison_lock_);
240 Statistics audio_jitter_buffer_ms_ RTC_GUARDED_BY(comparison_lock_);
241 Statistics pixels_ RTC_GUARDED_BY(comparison_lock_);
Sebastian Janssond4c5d632018-07-10 12:57:37 +0200242 // Rendered frame with worst PSNR is saved for further analysis.
243 absl::optional<FrameWithPsnr> worst_frame_ RTC_GUARDED_BY(comparison_lock_);
Elad Alon8c513c72019-05-07 21:22:24 +0200244 // Freeze metrics.
245 Statistics time_between_freezes_ RTC_GUARDED_BY(comparison_lock_);
246 uint32_t freeze_count_ RTC_GUARDED_BY(comparison_lock_);
247 uint32_t total_freezes_duration_ms_ RTC_GUARDED_BY(comparison_lock_);
248 uint32_t total_frames_duration_ms_ RTC_GUARDED_BY(comparison_lock_);
249 double sum_squared_frame_durations_ RTC_GUARDED_BY(comparison_lock_);
Sebastian Janssond4c5d632018-07-10 12:57:37 +0200250
Elad Alon58e06572019-05-08 15:34:24 +0200251 double decode_frame_rate_ RTC_GUARDED_BY(comparison_lock_);
252 double render_frame_rate_ RTC_GUARDED_BY(comparison_lock_);
253
Sebastian Janssond4c5d632018-07-10 12:57:37 +0200254 size_t last_fec_bytes_;
255
256 const int frames_to_process_;
257 int frames_recorded_;
258 int frames_processed_;
259 int dropped_frames_;
Ilya Nikolaevskiy6957abe2019-01-29 16:33:04 +0100260 int captured_frames_;
Sebastian Janssond4c5d632018-07-10 12:57:37 +0200261 int dropped_frames_before_first_encode_;
262 int dropped_frames_before_rendering_;
263 int64_t last_render_time_;
264 int64_t last_render_delta_ms_;
265 int64_t last_unfreeze_time_ms_;
266 uint32_t rtp_timestamp_delta_;
267 int64_t total_media_bytes_;
268 int64_t first_sending_time_;
269 int64_t last_sending_time_;
270
271 rtc::CriticalSection cpu_measurement_lock_;
272 int64_t cpu_time_ RTC_GUARDED_BY(cpu_measurement_lock_);
273 int64_t wallclock_time_ RTC_GUARDED_BY(cpu_measurement_lock_);
274
275 rtc::CriticalSection crit_;
276 std::deque<VideoFrame> frames_ RTC_GUARDED_BY(crit_);
277 absl::optional<VideoFrame> last_rendered_frame_ RTC_GUARDED_BY(crit_);
278 rtc::TimestampWrapAroundHandler wrap_handler_ RTC_GUARDED_BY(crit_);
279 std::map<int64_t, int64_t> send_times_ RTC_GUARDED_BY(crit_);
280 std::map<int64_t, int64_t> recv_times_ RTC_GUARDED_BY(crit_);
281 std::map<int64_t, size_t> encoded_frame_sizes_ RTC_GUARDED_BY(crit_);
282 absl::optional<uint32_t> first_encoded_timestamp_ RTC_GUARDED_BY(crit_);
283 absl::optional<uint32_t> first_sent_timestamp_ RTC_GUARDED_BY(crit_);
284 const double avg_psnr_threshold_;
285 const double avg_ssim_threshold_;
286 bool is_quick_test_enabled_;
287
288 std::vector<rtc::PlatformThread*> comparison_thread_pool_;
Sebastian Janssond4c5d632018-07-10 12:57:37 +0200289 rtc::Event comparison_available_event_;
290 std::deque<FrameComparison> comparisons_ RTC_GUARDED_BY(comparison_lock_);
Niels Möller4731f002019-05-03 09:34:24 +0200291 bool quit_ RTC_GUARDED_BY(comparison_lock_);
Sebastian Janssond4c5d632018-07-10 12:57:37 +0200292 rtc::Event done_;
Artem Titovff7730d2019-04-02 13:46:53 +0200293 test::SingleThreadedTaskQueueForTesting::TaskId stats_polling_task_id_
294 RTC_GUARDED_BY(comparison_lock_);
295 bool stop_stats_poller_ RTC_GUARDED_BY(comparison_lock_);
Sebastian Janssond4c5d632018-07-10 12:57:37 +0200296
297 std::unique_ptr<test::RtpFileWriter> rtp_file_writer_;
298 Clock* const clock_;
299 const int64_t start_ms_;
Artem Titovff7730d2019-04-02 13:46:53 +0200300 test::SingleThreadedTaskQueueForTesting* task_queue_;
Sebastian Janssond4c5d632018-07-10 12:57:37 +0200301};
302
303} // namespace webrtc
304#endif // VIDEO_VIDEO_ANALYZER_H_