blob: dda55b8030b1d86ddeb667bfb8b9d1abbf0fd9f3 [file] [log] [blame]
vspasova@webrtc.orgfd800702012-08-16 14:07:02 +00001/*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "rtc_tools/frame_analyzer/video_quality_analysis.h"
vspasova@webrtc.orgfd800702012-08-16 14:07:02 +000012
pbos@webrtc.org12dc1a32013-08-05 16:22:53 +000013#include <assert.h>
14#include <stdio.h>
15#include <stdlib.h>
mandermo74568172017-01-17 03:24:57 -080016#include <algorithm>
mandermo74568172017-01-17 03:24:57 -080017#include <map>
Yves Gerey665174f2018-06-19 15:03:05 +020018#include <string>
mandermo74568172017-01-17 03:24:57 -080019#include <utility>
vspasova@webrtc.orgfd800702012-08-16 14:07:02 +000020
Edward Lemur2e5966b2018-01-30 15:33:02 +010021#include "test/testsupport/perf_test.h"
Magnus Jedvert404be7f2018-08-22 19:23:34 +020022#include "third_party/libyuv/include/libyuv/compare.h"
23#include "third_party/libyuv/include/libyuv/convert.h"
Edward Lemur2e5966b2018-01-30 15:33:02 +010024
vspasova@webrtc.orgfd800702012-08-16 14:07:02 +000025#define STATS_LINE_LENGTH 32
26
27namespace webrtc {
28namespace test {
29
Henrik Kjellander67bcb602015-10-07 08:42:52 +020030ResultsContainer::ResultsContainer() {}
31ResultsContainer::~ResultsContainer() {}
32
vspasova@webrtc.orgac410e22012-08-27 14:57:19 +000033int GetI420FrameSize(int width, int height) {
34 int half_width = (width + 1) >> 1;
35 int half_height = (height + 1) >> 1;
vspasova@webrtc.orgfd800702012-08-16 14:07:02 +000036
Yves Gerey665174f2018-06-19 15:03:05 +020037 int y_plane = width * height; // I420 Y plane.
vspasova@webrtc.orgac410e22012-08-27 14:57:19 +000038 int u_plane = half_width * half_height; // I420 U plane.
39 int v_plane = half_width * half_height; // I420 V plane.
vspasova@webrtc.orgfd800702012-08-16 14:07:02 +000040
41 return y_plane + u_plane + v_plane;
42}
43
44int ExtractFrameSequenceNumber(std::string line) {
kjellander@webrtc.orgb2d74972013-01-26 16:36:40 +000045 size_t space_position = line.find(' ');
ehmaldonado1dffc622017-02-02 08:10:00 -080046 if (space_position == std::string::npos) {
vspasova@webrtc.orgfd800702012-08-16 14:07:02 +000047 return -1;
48 }
49 std::string frame = line.substr(0, space_position);
50
kjellander@webrtc.orgb2d74972013-01-26 16:36:40 +000051 size_t underscore_position = frame.find('_');
ehmaldonado1dffc622017-02-02 08:10:00 -080052 if (underscore_position == std::string::npos) {
vspasova@webrtc.orgfd800702012-08-16 14:07:02 +000053 return -1;
54 }
55 std::string frame_number = frame.substr(underscore_position + 1);
56
57 return strtol(frame_number.c_str(), NULL, 10);
58}
59
60int ExtractDecodedFrameNumber(std::string line) {
kjellander@webrtc.orgb2d74972013-01-26 16:36:40 +000061 size_t space_position = line.find(' ');
ehmaldonado1dffc622017-02-02 08:10:00 -080062 if (space_position == std::string::npos) {
vspasova@webrtc.orgfd800702012-08-16 14:07:02 +000063 return -1;
64 }
65 std::string decoded_number = line.substr(space_position + 1);
66
67 return strtol(decoded_number.c_str(), NULL, 10);
68}
69
70bool IsThereBarcodeError(std::string line) {
kjellander@webrtc.orgb2d74972013-01-26 16:36:40 +000071 size_t barcode_error_position = line.find("Barcode error");
ehmaldonado1dffc622017-02-02 08:10:00 -080072 if (barcode_error_position != std::string::npos) {
vspasova@webrtc.orgfd800702012-08-16 14:07:02 +000073 return true;
74 }
75 return false;
76}
77
78bool GetNextStatsLine(FILE* stats_file, char* line) {
79 int chars = 0;
80 char buf = 0;
81
82 while (buf != '\n') {
83 size_t chars_read = fread(&buf, 1, 1, stats_file);
84 if (chars_read != 1 || feof(stats_file)) {
85 return false;
86 }
87 line[chars] = buf;
88 ++chars;
89 }
Yves Gerey665174f2018-06-19 15:03:05 +020090 line[chars - 1] = '\0'; // Strip the trailing \n and put end of string.
vspasova@webrtc.orgfd800702012-08-16 14:07:02 +000091 return true;
92}
93
Magnus Jedvert404be7f2018-08-22 19:23:34 +020094template <typename FrameMetricFunction>
95static double CalculateMetric(
96 const FrameMetricFunction& frame_metric_function,
97 const rtc::scoped_refptr<I420BufferInterface>& ref_buffer,
98 const rtc::scoped_refptr<I420BufferInterface>& test_buffer) {
99 RTC_CHECK_EQ(ref_buffer->width(), test_buffer->width());
100 RTC_CHECK_EQ(ref_buffer->height(), test_buffer->height());
101 return frame_metric_function(
102 ref_buffer->DataY(), ref_buffer->StrideY(), ref_buffer->DataU(),
103 ref_buffer->StrideU(), ref_buffer->DataV(), ref_buffer->StrideV(),
104 test_buffer->DataY(), test_buffer->StrideY(), test_buffer->DataU(),
105 test_buffer->StrideU(), test_buffer->DataV(), test_buffer->StrideV(),
106 test_buffer->width(), test_buffer->height());
vspasova@webrtc.orgfd800702012-08-16 14:07:02 +0000107}
108
Magnus Jedvert404be7f2018-08-22 19:23:34 +0200109double Psnr(const rtc::scoped_refptr<I420BufferInterface>& ref_buffer,
110 const rtc::scoped_refptr<I420BufferInterface>& test_buffer) {
111 // LibYuv sets the max psnr value to 128, we restrict it to 48.
112 // In case of 0 mse in one frame, 128 can skew the results significantly.
113 return std::min(48.0,
114 CalculateMetric(&libyuv::I420Psnr, ref_buffer, test_buffer));
mcasas@webrtc.org6e2d0122014-03-14 12:45:45 +0000115}
116
Magnus Jedvert404be7f2018-08-22 19:23:34 +0200117double Ssim(const rtc::scoped_refptr<I420BufferInterface>& ref_buffer,
118 const rtc::scoped_refptr<I420BufferInterface>& test_buffer) {
119 return CalculateMetric(&libyuv::I420Ssim, ref_buffer, test_buffer);
vspasova@webrtc.orgfd800702012-08-16 14:07:02 +0000120}
121
Magnus Jedvert404be7f2018-08-22 19:23:34 +0200122void RunAnalysis(const rtc::scoped_refptr<webrtc::test::Video>& reference_video,
123 const rtc::scoped_refptr<webrtc::test::Video>& test_video,
mandermo74568172017-01-17 03:24:57 -0800124 const char* stats_file_reference_name,
125 const char* stats_file_test_name,
126 int width,
127 int height,
vspasova@webrtc.orgfd800702012-08-16 14:07:02 +0000128 ResultsContainer* results) {
mandermo74568172017-01-17 03:24:57 -0800129 FILE* stats_file_ref = fopen(stats_file_reference_name, "r");
130 FILE* stats_file_test = fopen(stats_file_test_name, "r");
vspasova@webrtc.orgfd800702012-08-16 14:07:02 +0000131
132 // String buffer for the lines in the stats file.
133 char line[STATS_LINE_LENGTH];
134
vspasova@webrtc.orgfd800702012-08-16 14:07:02 +0000135 int previous_frame_number = -1;
136
mandermo74568172017-01-17 03:24:57 -0800137 // Maps barcode id to the frame id for the reference video.
138 // In case two frames have same id, then we only save the first one.
139 std::map<int, int> ref_barcode_to_frame;
vspasova@webrtc.orgfd800702012-08-16 14:07:02 +0000140 // While there are entries in the stats file.
mandermo74568172017-01-17 03:24:57 -0800141 while (GetNextStatsLine(stats_file_ref, line)) {
142 int extracted_ref_frame = ExtractFrameSequenceNumber(line);
143 int decoded_frame_number = ExtractDecodedFrameNumber(line);
144
145 // Insert will only add if it is not in map already.
146 ref_barcode_to_frame.insert(
147 std::make_pair(decoded_frame_number, extracted_ref_frame));
148 }
149
150 while (GetNextStatsLine(stats_file_test, line)) {
vspasova@webrtc.orgfd800702012-08-16 14:07:02 +0000151 int extracted_test_frame = ExtractFrameSequenceNumber(line);
152 int decoded_frame_number = ExtractDecodedFrameNumber(line);
mandermo74568172017-01-17 03:24:57 -0800153 auto it = ref_barcode_to_frame.find(decoded_frame_number);
154 if (it == ref_barcode_to_frame.end()) {
155 // Not found in the reference video.
156 // TODO(mandermo) print
157 continue;
158 }
159 int extracted_ref_frame = it->second;
vspasova@webrtc.orgfd800702012-08-16 14:07:02 +0000160
161 // If there was problem decoding the barcode in this frame or the frame has
162 // been duplicated, continue.
163 if (IsThereBarcodeError(line) ||
164 decoded_frame_number == previous_frame_number) {
165 continue;
166 }
167
168 assert(extracted_test_frame != -1);
169 assert(decoded_frame_number != -1);
170
Magnus Jedvert404be7f2018-08-22 19:23:34 +0200171 const rtc::scoped_refptr<webrtc::I420BufferInterface> test_frame =
172 test_video->GetFrame(extracted_test_frame);
173 const rtc::scoped_refptr<webrtc::I420BufferInterface> reference_frame =
174 reference_video->GetFrame(extracted_ref_frame);
vspasova@webrtc.orgfd800702012-08-16 14:07:02 +0000175
176 // Calculate the PSNR and SSIM.
Magnus Jedvert404be7f2018-08-22 19:23:34 +0200177 double result_psnr = Psnr(reference_frame, test_frame);
178 double result_ssim = Ssim(reference_frame, test_frame);
vspasova@webrtc.orgfd800702012-08-16 14:07:02 +0000179
180 previous_frame_number = decoded_frame_number;
181
182 // Fill in the result struct.
183 AnalysisResult result;
184 result.frame_number = decoded_frame_number;
185 result.psnr_value = result_psnr;
186 result.ssim_value = result_ssim;
187
188 results->frames.push_back(result);
189 }
190
191 // Cleanup.
mandermo74568172017-01-17 03:24:57 -0800192 fclose(stats_file_ref);
193 fclose(stats_file_test);
vspasova@webrtc.orgfd800702012-08-16 14:07:02 +0000194}
195
mandermo7cebe782017-02-16 01:36:43 -0800196std::vector<std::pair<int, int> > CalculateFrameClusters(
197 FILE* file,
198 int* num_decode_errors) {
199 if (num_decode_errors) {
200 *num_decode_errors = 0;
201 }
mandermo74568172017-01-17 03:24:57 -0800202 std::vector<std::pair<int, int> > frame_cnt;
vspasova@webrtc.orgfd800702012-08-16 14:07:02 +0000203 char line[STATS_LINE_LENGTH];
mandermo74568172017-01-17 03:24:57 -0800204 while (GetNextStatsLine(file, line)) {
mandermo7cebe782017-02-16 01:36:43 -0800205 int decoded_frame_number;
206 if (IsThereBarcodeError(line)) {
207 decoded_frame_number = DECODE_ERROR;
208 if (num_decode_errors) {
209 ++*num_decode_errors;
210 }
211 } else {
212 decoded_frame_number = ExtractDecodedFrameNumber(line);
vspasova@webrtc.orgfd800702012-08-16 14:07:02 +0000213 }
mandermo7cebe782017-02-16 01:36:43 -0800214 if (frame_cnt.size() >= 2 && decoded_frame_number != DECODE_ERROR &&
215 frame_cnt.back().first == DECODE_ERROR &&
216 frame_cnt[frame_cnt.size() - 2].first == decoded_frame_number) {
217 // Handle when there is a decoding error inside a cluster of frames.
218 frame_cnt[frame_cnt.size() - 2].second += frame_cnt.back().second + 1;
219 frame_cnt.pop_back();
220 } else if (frame_cnt.empty() ||
221 frame_cnt.back().first != decoded_frame_number) {
mandermo74568172017-01-17 03:24:57 -0800222 frame_cnt.push_back(std::make_pair(decoded_frame_number, 1));
vspasova@webrtc.orgfd800702012-08-16 14:07:02 +0000223 } else {
mandermo74568172017-01-17 03:24:57 -0800224 ++frame_cnt.back().second;
vspasova@webrtc.orgfd800702012-08-16 14:07:02 +0000225 }
vspasova@webrtc.orgfd800702012-08-16 14:07:02 +0000226 }
mandermo74568172017-01-17 03:24:57 -0800227 return frame_cnt;
228}
mandermo74568172017-01-17 03:24:57 -0800229
Edward Lemur2e5966b2018-01-30 15:33:02 +0100230void GetMaxRepeatedAndSkippedFrames(const std::string& stats_file_ref_name,
231 const std::string& stats_file_test_name,
232 ResultsContainer* results) {
mandermo74568172017-01-17 03:24:57 -0800233 FILE* stats_file_ref = fopen(stats_file_ref_name.c_str(), "r");
234 FILE* stats_file_test = fopen(stats_file_test_name.c_str(), "r");
235 if (stats_file_ref == NULL) {
236 fprintf(stderr, "Couldn't open reference stats file for reading: %s\n",
237 stats_file_ref_name.c_str());
238 return;
239 }
240 if (stats_file_test == NULL) {
241 fprintf(stderr, "Couldn't open test stats file for reading: %s\n",
242 stats_file_test_name.c_str());
243 fclose(stats_file_ref);
244 return;
245 }
246
247 int max_repeated_frames = 1;
mandermo7cebe782017-02-16 01:36:43 -0800248 int max_skipped_frames = 0;
249
250 int decode_errors_ref = 0;
251 int decode_errors_test = 0;
mandermo74568172017-01-17 03:24:57 -0800252
253 std::vector<std::pair<int, int> > frame_cnt_ref =
mandermo7cebe782017-02-16 01:36:43 -0800254 CalculateFrameClusters(stats_file_ref, &decode_errors_ref);
mandermo74568172017-01-17 03:24:57 -0800255
256 std::vector<std::pair<int, int> > frame_cnt_test =
mandermo7cebe782017-02-16 01:36:43 -0800257 CalculateFrameClusters(stats_file_test, &decode_errors_test);
mandermo74568172017-01-17 03:24:57 -0800258
259 fclose(stats_file_ref);
260 fclose(stats_file_test);
261
262 auto it_ref = frame_cnt_ref.begin();
263 auto it_test = frame_cnt_test.begin();
264 auto end_ref = frame_cnt_ref.end();
265 auto end_test = frame_cnt_test.end();
266
267 if (it_test == end_test || it_ref == end_ref) {
268 fprintf(stderr, "Either test or ref file is empty, nothing to print\n");
269 return;
270 }
271
mandermo7cebe782017-02-16 01:36:43 -0800272 while (it_test != end_test && it_test->first == DECODE_ERROR) {
273 ++it_test;
274 }
275
276 if (it_test == end_test) {
277 fprintf(stderr, "Test video only has barcode decode errors\n");
278 return;
279 }
280
mandermo74568172017-01-17 03:24:57 -0800281 // Find the first frame in the reference video that match the first frame in
282 // the test video.
mandermo7cebe782017-02-16 01:36:43 -0800283 while (it_ref != end_ref &&
284 (it_ref->first == DECODE_ERROR || it_ref->first != it_test->first)) {
mandermo74568172017-01-17 03:24:57 -0800285 ++it_ref;
286 }
287 if (it_ref == end_ref) {
288 fprintf(stderr,
289 "The barcode in the test video's first frame is not in the "
290 "reference video.\n");
291 return;
292 }
293
mandermo7cebe782017-02-16 01:36:43 -0800294 int total_skipped_frames = 0;
mandermo74568172017-01-17 03:24:57 -0800295 for (;;) {
296 max_repeated_frames =
297 std::max(max_repeated_frames, it_test->second - it_ref->second + 1);
mandermo7cebe782017-02-16 01:36:43 -0800298
299 bool passed_error = false;
300
mandermo74568172017-01-17 03:24:57 -0800301 ++it_test;
mandermo7cebe782017-02-16 01:36:43 -0800302 while (it_test != end_test && it_test->first == DECODE_ERROR) {
303 ++it_test;
304 passed_error = true;
305 }
mandermo74568172017-01-17 03:24:57 -0800306 if (it_test == end_test) {
307 break;
308 }
mandermo7cebe782017-02-16 01:36:43 -0800309
mandermo74568172017-01-17 03:24:57 -0800310 int skipped_frames = 0;
311 ++it_ref;
mandermo7cebe782017-02-16 01:36:43 -0800312 for (; it_ref != end_ref; ++it_ref) {
313 if (it_ref->first != DECODE_ERROR && it_ref->first >= it_test->first) {
314 break;
315 }
316 ++skipped_frames;
mandermo74568172017-01-17 03:24:57 -0800317 }
mandermo7cebe782017-02-16 01:36:43 -0800318 if (passed_error) {
319 // If we pass an error in the test video, then we are conservative
320 // and will not calculate skipped frames for that part.
321 skipped_frames = 0;
mandermo74568172017-01-17 03:24:57 -0800322 }
mandermo7cebe782017-02-16 01:36:43 -0800323 if (it_ref != end_ref && it_ref->first == it_test->first) {
324 total_skipped_frames += skipped_frames;
325 if (skipped_frames > max_skipped_frames) {
326 max_skipped_frames = skipped_frames;
327 }
328 continue;
mandermo74568172017-01-17 03:24:57 -0800329 }
Edward Lemur2e5966b2018-01-30 15:33:02 +0100330 fprintf(stdout,
janssond4d2ece2017-05-30 07:48:06 -0700331 "Found barcode %d in test video, which is not in reference video\n",
janssoncc8b14b2017-05-23 08:31:11 -0700332 it_test->first);
janssond4d2ece2017-05-30 07:48:06 -0700333 break;
mandermo74568172017-01-17 03:24:57 -0800334 }
335
Edward Lemur2e5966b2018-01-30 15:33:02 +0100336 results->max_repeated_frames = max_repeated_frames;
337 results->max_skipped_frames = max_skipped_frames;
338 results->total_skipped_frames = total_skipped_frames;
339 results->decode_errors_ref = decode_errors_ref;
340 results->decode_errors_test = decode_errors_test;
vspasova@webrtc.orgfd800702012-08-16 14:07:02 +0000341}
342
kjellander@webrtc.orgf880f862013-09-10 12:10:01 +0000343void PrintAnalysisResults(const std::string& label, ResultsContainer* results) {
kjellander@webrtc.orge2df8b72013-11-03 18:34:51 +0000344 PrintAnalysisResults(stdout, label, results);
345}
346
Yves Gerey665174f2018-06-19 15:03:05 +0200347void PrintAnalysisResults(FILE* output,
348 const std::string& label,
kjellander@webrtc.orge2df8b72013-11-03 18:34:51 +0000349 ResultsContainer* results) {
Edward Lemur2e5966b2018-01-30 15:33:02 +0100350 SetPerfResultsOutput(output);
kjellander@webrtc.orgf880f862013-09-10 12:10:01 +0000351
352 if (results->frames.size() > 0u) {
Edward Lemur2e5966b2018-01-30 15:33:02 +0100353 PrintResult("Unique_frames_count", "", label, results->frames.size(),
354 "score", false);
kjellander@webrtc.orgf880f862013-09-10 12:10:01 +0000355
Edward Lemur2e5966b2018-01-30 15:33:02 +0100356 std::vector<double> psnr_values;
357 std::vector<double> ssim_values;
358 for (const auto& frame : results->frames) {
359 psnr_values.push_back(frame.psnr_value);
360 ssim_values.push_back(frame.ssim_value);
kjellander@webrtc.orgf880f862013-09-10 12:10:01 +0000361 }
Edward Lemur2e5966b2018-01-30 15:33:02 +0100362
363 PrintResultList("PSNR", "", label, psnr_values, "dB", false);
364 PrintResultList("SSIM", "", label, ssim_values, "score", false);
vspasova@webrtc.orgfd800702012-08-16 14:07:02 +0000365 }
Edward Lemur2e5966b2018-01-30 15:33:02 +0100366
367 PrintResult("Max_repeated", "", label, results->max_repeated_frames, "",
368 false);
369 PrintResult("Max_skipped", "", label, results->max_skipped_frames, "", false);
370 PrintResult("Total_skipped", "", label, results->total_skipped_frames, "",
371 false);
372 PrintResult("Decode_errors_reference", "", label, results->decode_errors_ref,
373 "", false);
374 PrintResult("Decode_errors_test", "", label, results->decode_errors_test, "",
375 false);
vspasova@webrtc.orgfd800702012-08-16 14:07:02 +0000376}
377
378} // namespace test
379} // namespace webrtc