Comparison of videos with reference frame not starting from zero
BUG=webrtc:6967
Review-Url: https://codereview.webrtc.org/2553693002
Cr-Commit-Position: refs/heads/master@{#16112}
diff --git a/webrtc/tools/compare_videos.py b/webrtc/tools/compare_videos.py
index 6aa659b..c4a70c2 100755
--- a/webrtc/tools/compare_videos.py
+++ b/webrtc/tools/compare_videos.py
@@ -48,15 +48,27 @@
help=('The path to where the zxing executable is located. '
'If omitted, it will be assumed to be present in the '
'PATH with the name zxing[.exe].'))
- parser.add_option('--stats_file', type='string', default='stats.txt',
+ parser.add_option('--stats_file_ref', type='string', default='stats_ref.txt',
help=('Path to the temporary stats file to be created and '
- 'used. Default: %default'))
+ 'used for the reference video file. '
+ 'Default: %default'))
+ parser.add_option('--stats_file_test', type='string',
+ default='stats_test.txt',
+ help=('Path to the temporary stats file to be created and '
+ 'used for the test video file. Default: %default'))
+ parser.add_option('--stats_file', type='string',
+ help=('DEPRECATED'))
parser.add_option('--yuv_frame_width', type='int', default=640,
help='Width of the YUV file\'s frames. Default: %default')
parser.add_option('--yuv_frame_height', type='int', default=480,
help='Height of the YUV file\'s frames. Default: %default')
options, _ = parser.parse_args()
+ if options.stats_file:
+ options.stats_file_test = options.stats_file
+ print ('WARNING: Using deprecated switch --stats_file. '
+ 'The new flag is --stats_file_test.')
+
if not options.ref_video:
parser.error('You must provide a path to the reference video!')
if not os.path.exists(options.ref_video):
@@ -74,6 +86,40 @@
options.frame_analyzer)
return options
+def _DevNull():
+ """On Windows, sometimes the inherited stdin handle from the parent process
+ fails. Workaround this by passing null to stdin to the subprocesses commands.
+ This function can be used to create the null file handler.
+ """
+ return open(os.devnull, 'r')
+
+def DecodeBarcodesInVideo(options, path_to_decoder, video, stat_file):
+ # Run barcode decoder on the test video to identify frame numbers.
+ png_working_directory = tempfile.mkdtemp()
+ cmd = [
+ sys.executable,
+ path_to_decoder,
+ '--yuv_file=%s' % video,
+ '--yuv_frame_width=%d' % options.yuv_frame_width,
+ '--yuv_frame_height=%d' % options.yuv_frame_height,
+ '--stats_file=%s' % stat_file,
+ '--png_working_dir=%s' % png_working_directory,
+ ]
+ if options.zxing_path:
+ cmd.append('--zxing_path=%s' % options.zxing_path)
+ if options.ffmpeg_path:
+ cmd.append('--ffmpeg_path=%s' % options.ffmpeg_path)
+
+
+ barcode_decoder = subprocess.Popen(cmd, stdin=_DevNull(),
+ stdout=sys.stdout, stderr=sys.stderr)
+ barcode_decoder.wait()
+
+ shutil.rmtree(png_working_directory)
+ if barcode_decoder.returncode != 0:
+ print 'Failed to run barcode decoder script.'
+ return 1
+ return 0
def main():
"""The main function.
@@ -97,32 +143,11 @@
path_to_decoder = os.path.join(SCRIPT_DIR, 'barcode_tools',
'barcode_decoder.py')
- # On Windows, sometimes the inherited stdin handle from the parent process
- # fails. Work around this by passing null to stdin to the subprocesses.
- null_filehandle = open(os.devnull, 'r')
-
- # Run barcode decoder on the test video to identify frame numbers.
- png_working_directory = tempfile.mkdtemp()
- cmd = [
- sys.executable,
- path_to_decoder,
- '--yuv_file=%s' % options.test_video,
- '--yuv_frame_width=%d' % options.yuv_frame_width,
- '--yuv_frame_height=%d' % options.yuv_frame_height,
- '--stats_file=%s' % options.stats_file,
- '--png_working_dir=%s' % png_working_directory,
- ]
- if options.zxing_path:
- cmd.append('--zxing_path=%s' % options.zxing_path)
- if options.ffmpeg_path:
- cmd.append('--ffmpeg_path=%s' % options.ffmpeg_path)
- barcode_decoder = subprocess.Popen(cmd, stdin=null_filehandle,
- stdout=sys.stdout, stderr=sys.stderr)
- barcode_decoder.wait()
-
- shutil.rmtree(png_working_directory)
- if barcode_decoder.returncode != 0:
- print 'Failed to run barcode decoder script.'
+ if DecodeBarcodesInVideo(options, path_to_decoder,
+ options.ref_video, options.stats_file_ref) != 0:
+ return 1
+ if DecodeBarcodesInVideo(options, path_to_decoder,
+ options.test_video, options.stats_file_test) != 0:
return 1
# Run frame analyzer to compare the videos and print output.
@@ -131,11 +156,12 @@
'--label=%s' % options.label,
'--reference_file=%s' % options.ref_video,
'--test_file=%s' % options.test_video,
- '--stats_file=%s' % options.stats_file,
+ '--stats_file_ref=%s' % options.stats_file_ref,
+ '--stats_file_test=%s' % options.stats_file_test,
'--width=%d' % options.yuv_frame_width,
'--height=%d' % options.yuv_frame_height,
]
- frame_analyzer = subprocess.Popen(cmd, stdin=null_filehandle,
+ frame_analyzer = subprocess.Popen(cmd, stdin=_DevNull(),
stdout=sys.stdout, stderr=sys.stderr)
frame_analyzer.wait()
if frame_analyzer.returncode != 0:
diff --git a/webrtc/tools/frame_analyzer/frame_analyzer.cc b/webrtc/tools/frame_analyzer/frame_analyzer.cc
index 8020109..0a3be19 100644
--- a/webrtc/tools/frame_analyzer/frame_analyzer.cc
+++ b/webrtc/tools/frame_analyzer/frame_analyzer.cc
@@ -23,12 +23,11 @@
* video. The test video is a record of the reference video which can start at
* an arbitrary point. It is possible that there will be repeated frames or
* skipped frames as well. In order to have a way to compare corresponding
- * frames from the two videos, a stats file should be provided. The stats file
+ * frames from the two videos, two stats files should be provided. One for the
+ * reference video and one for the test video. The stats file
* is a text file assumed to be in the format:
- * frame_xxxx yyyy
- * where xxxx is the frame number in the test video and yyyy is the
- * corresponding frame number in the original video.
- * The video files should be 1420 YUV videos.
+ * frame_xxxx yyyy where xxxx is the frame number in and yyyy is the
+ * corresponding barcode. The video files should be 1420 YUV videos.
* The tool prints the result to standard output in the Chromium perf format:
* RESULT <metric>:<label>= <values>
*
@@ -36,22 +35,30 @@
*
* Usage:
* frame_analyzer --label=<test_label> --reference_file=<name_of_file>
- * --test_file=<name_of_file> --stats_file=<name_of_file> --width=<frame_width>
+ * --test_file_ref=<name_of_file> --stats_file_test=<name_of_file>
+ * --stats_file=<name_of_file> --width=<frame_width>
* --height=<frame_height>
*/
int main(int argc, char** argv) {
std::string program_name = argv[0];
- std::string usage = "Compares the output video with the initially sent video."
- "\nExample usage:\n" + program_name + " --stats_file=stats.txt "
- "--reference_file=ref.yuv --test_file=test.yuv --width=320 --height=240\n"
+ std::string usage =
+ "Compares the output video with the initially sent video."
+ "\nExample usage:\n" +
+ program_name +
+ " --reference_file=ref.yuv --test_file=test.yuv --width=320 "
+ "--height=240\n"
"Command line flags:\n"
" - width(int): The width of the reference and test files. Default: -1\n"
" - height(int): The height of the reference and test files. "
" Default: -1\n"
" - label(string): The label to use for the perf output."
" Default: MY_TEST\n"
- " - stats_file(string): The full name of the file containing the stats"
- " after decoding of the received YUV video. Default: stats.txt\n"
+ " - stats_file_ref(string): The path to the stats file that will be"
+ " produced for the reference video file."
+ " Default: stats_ref.txt\n"
+ " - stats_file_test(string): The path to the stats file that will be"
+ " produced for the test video file."
+ " Default: stats_test.txt\n"
" - reference_file(string): The reference YUV file to compare against."
" Default: ref.yuv\n"
" - test_file(string): The test YUV file to run the analysis for."
@@ -66,7 +73,8 @@
parser.SetFlag("width", "-1");
parser.SetFlag("height", "-1");
parser.SetFlag("label", "MY_TEST");
- parser.SetFlag("stats_file", "stats.txt");
+ parser.SetFlag("stats_file_ref", "stats_ref.txt");
+ parser.SetFlag("stats_file_test", "stats_test.txt");
parser.SetFlag("reference_file", "ref.yuv");
parser.SetFlag("test_file", "test.yuv");
parser.SetFlag("help", "false");
@@ -90,11 +98,13 @@
webrtc::test::RunAnalysis(parser.GetFlag("reference_file").c_str(),
parser.GetFlag("test_file").c_str(),
- parser.GetFlag("stats_file").c_str(), width, height,
- &results);
+ parser.GetFlag("stats_file_ref").c_str(),
+ parser.GetFlag("stats_file_test").c_str(), width,
+ height, &results);
std::string label = parser.GetFlag("label");
webrtc::test::PrintAnalysisResults(label, &results);
- webrtc::test::PrintMaxRepeatedAndSkippedFrames(label,
- parser.GetFlag("stats_file"));
+ webrtc::test::PrintMaxRepeatedAndSkippedFrames(
+ label, parser.GetFlag("stats_file_ref"),
+ parser.GetFlag("stats_file_test"));
}
diff --git a/webrtc/tools/frame_analyzer/video_quality_analysis.cc b/webrtc/tools/frame_analyzer/video_quality_analysis.cc
index c1911ca..dfb7d90 100644
--- a/webrtc/tools/frame_analyzer/video_quality_analysis.cc
+++ b/webrtc/tools/frame_analyzer/video_quality_analysis.cc
@@ -13,7 +13,10 @@
#include <assert.h>
#include <stdio.h>
#include <stdlib.h>
+#include <algorithm>
#include <string>
+#include <map>
+#include <utility>
#define STATS_LINE_LENGTH 32
#define Y4M_FILE_HEADER_MAX_SIZE 200
@@ -224,8 +227,12 @@
return result;
}
-void RunAnalysis(const char* reference_file_name, const char* test_file_name,
- const char* stats_file_name, int width, int height,
+void RunAnalysis(const char* reference_file_name,
+ const char* test_file_name,
+ const char* stats_file_reference_name,
+ const char* stats_file_test_name,
+ int width,
+ int height,
ResultsContainer* results) {
// Check if the reference_file_name ends with "y4m".
bool y4m_mode = false;
@@ -234,7 +241,8 @@
}
int size = GetI420FrameSize(width, height);
- FILE* stats_file = fopen(stats_file_name, "r");
+ FILE* stats_file_ref = fopen(stats_file_reference_name, "r");
+ FILE* stats_file_test = fopen(stats_file_test_name, "r");
// String buffer for the lines in the stats file.
char line[STATS_LINE_LENGTH];
@@ -244,10 +252,29 @@
uint8_t* reference_frame = new uint8_t[size];
int previous_frame_number = -1;
+ // Maps barcode id to the frame id for the reference video.
+ // In case two frames have same id, then we only save the first one.
+ std::map<int, int> ref_barcode_to_frame;
// While there are entries in the stats file.
- while (GetNextStatsLine(stats_file, line)) {
+ while (GetNextStatsLine(stats_file_ref, line)) {
+ int extracted_ref_frame = ExtractFrameSequenceNumber(line);
+ int decoded_frame_number = ExtractDecodedFrameNumber(line);
+
+ // Insert will only add if it is not in map already.
+ ref_barcode_to_frame.insert(
+ std::make_pair(decoded_frame_number, extracted_ref_frame));
+ }
+
+ while (GetNextStatsLine(stats_file_test, line)) {
int extracted_test_frame = ExtractFrameSequenceNumber(line);
int decoded_frame_number = ExtractDecodedFrameNumber(line);
+ auto it = ref_barcode_to_frame.find(decoded_frame_number);
+ if (it == ref_barcode_to_frame.end()) {
+ // Not found in the reference video.
+ // TODO(mandermo) print
+ continue;
+ }
+ int extracted_ref_frame = it->second;
// If there was problem decoding the barcode in this frame or the frame has
// been duplicated, continue.
@@ -263,17 +290,17 @@
test_frame);
if (y4m_mode) {
ExtractFrameFromY4mFile(reference_file_name, width, height,
- decoded_frame_number, reference_frame);
+ extracted_ref_frame, reference_frame);
} else {
ExtractFrameFromYuvFile(reference_file_name, width, height,
- decoded_frame_number, reference_frame);
+ extracted_ref_frame, reference_frame);
}
// Calculate the PSNR and SSIM.
- double result_psnr = CalculateMetrics(kPSNR, reference_frame, test_frame,
- width, height);
- double result_ssim = CalculateMetrics(kSSIM, reference_frame, test_frame,
- width, height);
+ double result_psnr =
+ CalculateMetrics(kPSNR, reference_frame, test_frame, width, height);
+ double result_ssim =
+ CalculateMetrics(kSSIM, reference_frame, test_frame, width, height);
previous_frame_number = decoded_frame_number;
@@ -287,62 +314,124 @@
}
// Cleanup.
- fclose(stats_file);
+ fclose(stats_file_ref);
+ fclose(stats_file_test);
delete[] test_frame;
delete[] reference_frame;
}
void PrintMaxRepeatedAndSkippedFrames(const std::string& label,
- const std::string& stats_file_name) {
- PrintMaxRepeatedAndSkippedFrames(stdout, label, stats_file_name);
+ const std::string& stats_file_ref_name,
+ const std::string& stats_file_test_name) {
+ PrintMaxRepeatedAndSkippedFrames(stdout, label, stats_file_ref_name,
+ stats_file_test_name);
}
-void PrintMaxRepeatedAndSkippedFrames(FILE* output, const std::string& label,
- const std::string& stats_file_name) {
- FILE* stats_file = fopen(stats_file_name.c_str(), "r");
- if (stats_file == NULL) {
- fprintf(stderr, "Couldn't open stats file for reading: %s\n",
- stats_file_name.c_str());
- return;
- }
+namespace {
+// Clusters the frames in the file. First in the pair is the frame number and
+// second is the number
+// of frames in that cluster. So if first frame in video has number 100 and it
+// is repeated 3 after
+// each other, then the first entry in the returned vector has first set to 100
+// and second set
+// to 3.
+std::vector<std::pair<int, int> > CalculateFrameClusters(FILE* file) {
+ std::vector<std::pair<int, int> > frame_cnt;
char line[STATS_LINE_LENGTH];
-
- int repeated_frames = 1;
- int max_repeated_frames = 1;
- int max_skipped_frames = 1;
- int previous_frame_number = -1;
-
- while (GetNextStatsLine(stats_file, line)) {
+ while (GetNextStatsLine(file, line)) {
int decoded_frame_number = ExtractDecodedFrameNumber(line);
-
if (decoded_frame_number == -1) {
continue;
}
-
- // Calculate how many frames a cluster of repeated frames contains.
- if (decoded_frame_number == previous_frame_number) {
- ++repeated_frames;
- if (repeated_frames > max_repeated_frames) {
- max_repeated_frames = repeated_frames;
- }
+ if (frame_cnt.empty() || frame_cnt.back().first != decoded_frame_number) {
+ frame_cnt.push_back(std::make_pair(decoded_frame_number, 1));
} else {
- repeated_frames = 1;
+ ++frame_cnt.back().second;
}
-
- // Calculate how much frames have been skipped.
- if (decoded_frame_number != 0 && previous_frame_number != -1) {
- int skipped_frames = decoded_frame_number - previous_frame_number - 1;
- if (skipped_frames > max_skipped_frames) {
- max_skipped_frames = skipped_frames;
- }
- }
- previous_frame_number = decoded_frame_number;
}
+ return frame_cnt;
+}
+} // namespace
+
+void PrintMaxRepeatedAndSkippedFrames(FILE* output,
+ const std::string& label,
+ const std::string& stats_file_ref_name,
+ const std::string& stats_file_test_name) {
+ FILE* stats_file_ref = fopen(stats_file_ref_name.c_str(), "r");
+ FILE* stats_file_test = fopen(stats_file_test_name.c_str(), "r");
+ if (stats_file_ref == NULL) {
+ fprintf(stderr, "Couldn't open reference stats file for reading: %s\n",
+ stats_file_ref_name.c_str());
+ return;
+ }
+ if (stats_file_test == NULL) {
+ fprintf(stderr, "Couldn't open test stats file for reading: %s\n",
+ stats_file_test_name.c_str());
+ fclose(stats_file_ref);
+ return;
+ }
+
+ int max_repeated_frames = 1;
+ int max_skipped_frames = 1;
+
+ std::vector<std::pair<int, int> > frame_cnt_ref =
+ CalculateFrameClusters(stats_file_ref);
+
+ std::vector<std::pair<int, int> > frame_cnt_test =
+ CalculateFrameClusters(stats_file_test);
+
+ fclose(stats_file_ref);
+ fclose(stats_file_test);
+
+ auto it_ref = frame_cnt_ref.begin();
+ auto it_test = frame_cnt_test.begin();
+ auto end_ref = frame_cnt_ref.end();
+ auto end_test = frame_cnt_test.end();
+
+ if (it_test == end_test || it_ref == end_ref) {
+ fprintf(stderr, "Either test or ref file is empty, nothing to print\n");
+ return;
+ }
+
+ // Find the first frame in the reference video that match the first frame in
+ // the test video.
+ while (it_ref != end_ref && it_ref->first != it_test->first) {
+ ++it_ref;
+ }
+ if (it_ref == end_ref) {
+ fprintf(stderr,
+ "The barcode in the test video's first frame is not in the "
+ "reference video.\n");
+ return;
+ }
+
+ for (;;) {
+ max_repeated_frames =
+ std::max(max_repeated_frames, it_test->second - it_ref->second + 1);
+ ++it_test;
+ if (it_test == end_test) {
+ break;
+ }
+ int skipped_frames = 0;
+ ++it_ref;
+ while (it_ref != end_ref && it_ref->first != it_test->first) {
+ skipped_frames += it_ref->second;
+ ++it_ref;
+ }
+ if (it_ref == end_ref) {
+ fprintf(stderr,
+ "The barcode in the test video is not in the reference video.\n");
+ return;
+ }
+ if (skipped_frames > max_skipped_frames) {
+ max_skipped_frames = skipped_frames;
+ }
+ }
+
fprintf(output, "RESULT Max_repeated: %s= %d\n", label.c_str(),
max_repeated_frames);
fprintf(output, "RESULT Max_skipped: %s= %d\n", label.c_str(),
max_skipped_frames);
- fclose(stats_file);
}
void PrintAnalysisResults(const std::string& label, ResultsContainer* results) {
diff --git a/webrtc/tools/frame_analyzer/video_quality_analysis.h b/webrtc/tools/frame_analyzer/video_quality_analysis.h
index 475b2fa..93d3605 100644
--- a/webrtc/tools/frame_analyzer/video_quality_analysis.h
+++ b/webrtc/tools/frame_analyzer/video_quality_analysis.h
@@ -45,16 +45,22 @@
// There may be missing or duplicate frames. Also the frames start at a random
// position in the original video. We should provide a statistics file along
// with the test video. The stats file contains the connection between the
-// actual frames in the test file and their position in the reference video, so
-// that the analysis could run with the right frames from both videos. The stats
-// file should be in the form 'frame_xxxx yyyy', where xxxx is the consecutive
-// number of the frame in the test video, and yyyy is the equivalent frame in
-// the reference video. The stats file could be produced by
+// actual frames in the test file and their bar code number. There is one file
+// for the reference video and one for the test video. The stats file should
+// be in the form 'frame_xxxx yyyy', where xxxx is the consecutive
+// number of the frame in the test video, and yyyy is the barcode number.
+// The stats file could be produced by
// tools/barcode_tools/barcode_decoder.py. This script decodes the barcodes
// integrated in every video and generates the stats file. If three was some
// problem with the decoding there would be 'Barcode error' instead of yyyy.
-void RunAnalysis(const char* reference_file_name, const char* test_file_name,
- const char* stats_file_name, int width, int height,
+// The stat files are used to compare the right frames with each other and
+// to calculate statistics.
+void RunAnalysis(const char* reference_file_name,
+ const char* test_file_name,
+ const char* stats_file_reference_name,
+ const char* stats_file_test_name,
+ int width,
+ int height,
ResultsContainer* results);
// Compute PSNR or SSIM for an I420 frame (all planes). When we are calculating
@@ -79,11 +85,14 @@
// Calculates max repeated and skipped frames and prints them to stdout in a
// format that is compatible with Chromium performance numbers.
void PrintMaxRepeatedAndSkippedFrames(const std::string& label,
- const std::string& stats_file_name);
+ const std::string& stats_file_ref_name,
+ const std::string& stats_file_test_name);
// Similar to the above, but will print to the specified file handle.
-void PrintMaxRepeatedAndSkippedFrames(FILE* output, const std::string& label,
- const std::string& stats_file_name);
+void PrintMaxRepeatedAndSkippedFrames(FILE* output,
+ const std::string& label,
+ const std::string& stats_file_ref_name,
+ const std::string& stats_file_test_name);
// Gets the next line from an open stats file.
bool GetNextStatsLine(FILE* stats_file, char* line);
diff --git a/webrtc/tools/frame_analyzer/video_quality_analysis_unittest.cc b/webrtc/tools/frame_analyzer/video_quality_analysis_unittest.cc
index 34c5fa7..85d0b46 100644
--- a/webrtc/tools/frame_analyzer/video_quality_analysis_unittest.cc
+++ b/webrtc/tools/frame_analyzer/video_quality_analysis_unittest.cc
@@ -85,24 +85,42 @@
}
TEST_F(VideoQualityAnalysisTest, PrintMaxRepeatedAndSkippedFramesInvalidFile) {
- std::string stats_filename = OutputPath() + "non-existing-stats-file.txt";
+ std::string stats_filename_ref =
+ OutputPath() + "non-existing-stats-file-1.txt";
+ std::string stats_filename = OutputPath() + "non-existing-stats-file-2.txt";
remove(stats_filename.c_str());
PrintMaxRepeatedAndSkippedFrames(logfile_, "NonExistingStatsFile",
- stats_filename);
+ stats_filename_ref, stats_filename);
}
TEST_F(VideoQualityAnalysisTest,
PrintMaxRepeatedAndSkippedFramesEmptyStatsFile) {
- std::string stats_filename = OutputPath() + "empty-stats.txt";
+ std::string stats_filename_ref = OutputPath() + "empty-stats-1.txt";
+ std::string stats_filename = OutputPath() + "empty-stats-2.txt";
std::ofstream stats_file;
+ stats_file.open(stats_filename_ref.c_str());
+ stats_file.close();
stats_file.open(stats_filename.c_str());
stats_file.close();
- PrintMaxRepeatedAndSkippedFrames(logfile_, "EmptyStatsFile", stats_filename);
+ PrintMaxRepeatedAndSkippedFrames(logfile_, "EmptyStatsFile",
+ stats_filename_ref, stats_filename);
}
TEST_F(VideoQualityAnalysisTest, PrintMaxRepeatedAndSkippedFramesNormalFile) {
- std::string stats_filename = OutputPath() + "stats.txt";
+ std::string stats_filename_ref = OutputPath() + "stats-1.txt";
+ std::string stats_filename = OutputPath() + "stats-2.txt";
std::ofstream stats_file;
+
+ stats_file.open(stats_filename_ref.c_str());
+ stats_file << "frame_0001 0100\n";
+ stats_file << "frame_0002 0101\n";
+ stats_file << "frame_0003 0102\n";
+ stats_file << "frame_0004 0103\n";
+ stats_file << "frame_0005 0106\n";
+ stats_file << "frame_0006 0107\n";
+ stats_file << "frame_0007 0108\n";
+ stats_file.close();
+
stats_file.open(stats_filename.c_str());
stats_file << "frame_0001 0100\n";
stats_file << "frame_0002 0101\n";
@@ -110,7 +128,8 @@
stats_file << "frame_0004 0106\n";
stats_file.close();
- PrintMaxRepeatedAndSkippedFrames(logfile_, "NormalStatsFile", stats_filename);
+ PrintMaxRepeatedAndSkippedFrames(logfile_, "NormalStatsFile",
+ stats_filename_ref, stats_filename);
}