blob: f3808f88c18165fdfd3a56eb0ad4fa3306485755 [file] [log] [blame]
kjellander@webrtc.org35a17562011-10-06 06:44:54 +00001/*
marpan@webrtc.orgf4c2de92012-06-05 21:07:28 +00002 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
kjellander@webrtc.org35a17562011-10-06 06:44:54 +00003 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
kjellander@webrtc.org5b97b122011-12-08 07:42:18 +000010
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#ifndef MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_H_
12#define MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_H_
kjellander@webrtc.org35a17562011-10-06 06:44:54 +000013
brandtrb57f4262017-08-30 06:29:51 -070014#include <map>
Erik Språng08127a92016-11-16 16:41:30 +010015#include <memory>
kjellander@webrtc.org35a17562011-10-06 06:44:54 +000016#include <string>
brandtr17b958c2017-03-07 01:41:43 -080017#include <vector>
kjellander@webrtc.org35a17562011-10-06 06:44:54 +000018
Kári Tristan Helgason169005d2018-05-22 13:34:14 +020019#include "api/test/videocodec_test_fixture.h"
20#include "api/test/videocodec_test_stats.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020021#include "api/video/video_frame.h"
Rasmus Brandtd062a3c2018-03-08 16:45:54 +010022#include "common_video/include/video_bitrate_allocator.h"
Kári Tristan Helgason169005d2018-05-22 13:34:14 +020023#include "modules/video_coding/include/video_codec_interface.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020024#include "modules/video_coding/utility/ivf_file_writer.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020025#include "rtc_base/buffer.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020026#include "rtc_base/constructormagic.h"
27#include "rtc_base/sequenced_task_checker.h"
28#include "rtc_base/task_queue.h"
29#include "test/testsupport/frame_reader.h"
30#include "test/testsupport/frame_writer.h"
kjellander@webrtc.org35a17562011-10-06 06:44:54 +000031
32namespace webrtc {
33namespace test {
34
kjellander@webrtc.org35a17562011-10-06 06:44:54 +000035// Handles encoding/decoding of video using the VideoEncoder/VideoDecoder
36// interfaces. This is done in a sequential manner in order to be able to
37// measure times properly.
38// The class processes a frame at the time for the configured input file.
39// It maintains state of where in the source input file the processing is at.
kjellander@webrtc.org35a17562011-10-06 06:44:54 +000040class VideoProcessor {
41 public:
Sergey Silkin10d9d592018-02-01 13:25:17 +010042 using VideoDecoderList = std::vector<std::unique_ptr<VideoDecoder>>;
43 using IvfFileWriterList = std::vector<std::unique_ptr<IvfFileWriter>>;
44 using FrameWriterList = std::vector<std::unique_ptr<FrameWriter>>;
45
brandtrc4095522017-08-07 08:12:33 -070046 VideoProcessor(webrtc::VideoEncoder* encoder,
Sergey Silkin10d9d592018-02-01 13:25:17 +010047 VideoDecoderList* decoders,
48 FrameReader* input_frame_reader,
Kári Tristan Helgason169005d2018-05-22 13:34:14 +020049 const VideoCodecTestFixture::Config& config,
50 VideoCodecTestStats* stats,
Sergey Silkin10d9d592018-02-01 13:25:17 +010051 IvfFileWriterList* encoded_frame_writers,
52 FrameWriterList* decoded_frame_writers);
brandtrc4095522017-08-07 08:12:33 -070053 ~VideoProcessor();
kjellander@webrtc.org35a17562011-10-06 06:44:54 +000054
Sergey Silkin10d9d592018-02-01 13:25:17 +010055 // Reads a frame and sends it to the encoder. When the encode callback
56 // is received, the encoded frame is buffered. After encoding is finished
57 // buffered frame is sent to decoder. Quality evaluation is done in
58 // the decode callback.
brandtr8935d972017-09-06 01:53:22 -070059 void ProcessFrame();
marpan@webrtc.orgf4c2de92012-06-05 21:07:28 +000060
brandtrbdd555c2017-08-21 01:34:04 -070061 // Updates the encoder with target rates. Must be called at least once.
Sergey Silkin3be2a552018-01-17 15:11:44 +010062 void SetRates(size_t bitrate_kbps, size_t framerate_fps);
kjellander@webrtc.org35a17562011-10-06 06:44:54 +000063
64 private:
kjellander@webrtc.org35a17562011-10-06 06:44:54 +000065 class VideoProcessorEncodeCompleteCallback
pbos@webrtc.org7f7162a2013-07-30 15:18:31 +000066 : public webrtc::EncodedImageCallback {
kjellander@webrtc.org35a17562011-10-06 06:44:54 +000067 public:
brandtrc4095522017-08-07 08:12:33 -070068 explicit VideoProcessorEncodeCompleteCallback(
69 VideoProcessor* video_processor)
brandtrc8c59052017-08-21 06:44:16 -070070 : video_processor_(video_processor),
Rasmus Brandt4b381af2018-02-07 13:56:16 +010071 task_queue_(rtc::TaskQueue::Current()) {
Rasmus Brandt5f7a8912018-02-28 17:17:15 +010072 RTC_DCHECK(video_processor_);
Rasmus Brandt4b381af2018-02-07 13:56:16 +010073 RTC_DCHECK(task_queue_);
74 }
brandtrc8c59052017-08-21 06:44:16 -070075
Sergey Ulanov525df3f2016-08-02 17:46:41 -070076 Result OnEncodedImage(
pbos@webrtc.org273a4142014-12-01 15:23:21 +000077 const webrtc::EncodedImage& encoded_image,
78 const webrtc::CodecSpecificInfo* codec_specific_info,
brandtr8bc93852017-02-15 05:19:51 -080079 const webrtc::RTPFragmentationHeader* fragmentation) override {
brandtr8bc93852017-02-15 05:19:51 -080080 RTC_CHECK(codec_specific_info);
brandtrc8c59052017-08-21 06:44:16 -070081
Rasmus Brandt4b381af2018-02-07 13:56:16 +010082 // Post the callback to the right task queue, if needed.
83 if (!task_queue_->IsCurrent()) {
brandtr45535622017-08-22 03:33:11 -070084 task_queue_->PostTask(
85 std::unique_ptr<rtc::QueuedTask>(new EncodeCallbackTask(
86 video_processor_, encoded_image, codec_specific_info)));
brandtrc8c59052017-08-21 06:44:16 -070087 return Result(Result::OK, 0);
88 }
89
Sergey Silkin10d9d592018-02-01 13:25:17 +010090 video_processor_->FrameEncoded(encoded_image, *codec_specific_info);
brandtr8bc93852017-02-15 05:19:51 -080091 return Result(Result::OK, 0);
92 }
kjellander@webrtc.org35a17562011-10-06 06:44:54 +000093
94 private:
brandtrc8c59052017-08-21 06:44:16 -070095 class EncodeCallbackTask : public rtc::QueuedTask {
96 public:
97 EncodeCallbackTask(VideoProcessor* video_processor,
98 const webrtc::EncodedImage& encoded_image,
brandtr45535622017-08-22 03:33:11 -070099 const webrtc::CodecSpecificInfo* codec_specific_info)
brandtrc8c59052017-08-21 06:44:16 -0700100 : video_processor_(video_processor),
101 buffer_(encoded_image._buffer, encoded_image._length),
102 encoded_image_(encoded_image),
103 codec_specific_info_(*codec_specific_info) {
104 encoded_image_._buffer = buffer_.data();
brandtrc8c59052017-08-21 06:44:16 -0700105 }
106
107 bool Run() override {
Sergey Silkin10d9d592018-02-01 13:25:17 +0100108 video_processor_->FrameEncoded(encoded_image_, codec_specific_info_);
brandtrc8c59052017-08-21 06:44:16 -0700109 return true;
110 }
111
112 private:
113 VideoProcessor* const video_processor_;
114 rtc::Buffer buffer_;
115 webrtc::EncodedImage encoded_image_;
116 const webrtc::CodecSpecificInfo codec_specific_info_;
brandtrc8c59052017-08-21 06:44:16 -0700117 };
118
brandtrc4095522017-08-07 08:12:33 -0700119 VideoProcessor* const video_processor_;
brandtrc8c59052017-08-21 06:44:16 -0700120 rtc::TaskQueue* const task_queue_;
kjellander@webrtc.org35a17562011-10-06 06:44:54 +0000121 };
122
kjellander@webrtc.org35a17562011-10-06 06:44:54 +0000123 class VideoProcessorDecodeCompleteCallback
philipelcce46fc2015-12-21 03:04:49 -0800124 : public webrtc::DecodedImageCallback {
kjellander@webrtc.org35a17562011-10-06 06:44:54 +0000125 public:
brandtrc4095522017-08-07 08:12:33 -0700126 explicit VideoProcessorDecodeCompleteCallback(
Sergey Silkin645e2e02018-04-06 09:42:13 +0200127 VideoProcessor* video_processor,
128 size_t simulcast_svc_idx)
brandtrc8c59052017-08-21 06:44:16 -0700129 : video_processor_(video_processor),
Sergey Silkin645e2e02018-04-06 09:42:13 +0200130 simulcast_svc_idx_(simulcast_svc_idx),
Rasmus Brandt4b381af2018-02-07 13:56:16 +0100131 task_queue_(rtc::TaskQueue::Current()) {
Rasmus Brandt5f7a8912018-02-28 17:17:15 +0100132 RTC_DCHECK(video_processor_);
Rasmus Brandt4b381af2018-02-07 13:56:16 +0100133 RTC_DCHECK(task_queue_);
134 }
brandtrc8c59052017-08-21 06:44:16 -0700135
Sami Kalliomäki451b29c2018-07-04 14:33:51 +0200136 int32_t Decoded(webrtc::VideoFrame& image) override;
brandtrc8c59052017-08-21 06:44:16 -0700137
philipelcce46fc2015-12-21 03:04:49 -0800138 int32_t Decoded(webrtc::VideoFrame& image,
139 int64_t decode_time_ms) override {
brandtr6bb8e0e2017-02-20 04:35:52 -0800140 return Decoded(image);
kjellander@webrtc.org35a17562011-10-06 06:44:54 +0000141 }
brandtrc8c59052017-08-21 06:44:16 -0700142
brandtr6bb8e0e2017-02-20 04:35:52 -0800143 void Decoded(webrtc::VideoFrame& image,
Danil Chapovalov0040b662018-06-18 10:48:16 +0200144 absl::optional<int32_t> decode_time_ms,
145 absl::optional<uint8_t> qp) override {
brandtrbea36fd2017-08-07 03:36:54 -0700146 Decoded(image);
sakalcc452e12017-02-09 04:53:45 -0800147 }
kjellander@webrtc.org35a17562011-10-06 06:44:54 +0000148
149 private:
brandtrc4095522017-08-07 08:12:33 -0700150 VideoProcessor* const video_processor_;
Sergey Silkin645e2e02018-04-06 09:42:13 +0200151 const size_t simulcast_svc_idx_;
brandtrc8c59052017-08-21 06:44:16 -0700152 rtc::TaskQueue* const task_queue_;
kjellander@webrtc.org35a17562011-10-06 06:44:54 +0000153 };
brandtr8bc93852017-02-15 05:19:51 -0800154
brandtrbdd555c2017-08-21 01:34:04 -0700155 // Invoked by the callback adapter when a frame has completed encoding.
Sergey Silkin10d9d592018-02-01 13:25:17 +0100156 void FrameEncoded(const webrtc::EncodedImage& encoded_image,
157 const webrtc::CodecSpecificInfo& codec_specific);
brandtr8bc93852017-02-15 05:19:51 -0800158
brandtrbdd555c2017-08-21 01:34:04 -0700159 // Invoked by the callback adapter when a frame has completed decoding.
Sergey Silkin645e2e02018-04-06 09:42:13 +0200160 void FrameDecoded(const webrtc::VideoFrame& image, size_t simulcast_svc_idx);
161
162 void DecodeFrame(const EncodedImage& encoded_image, size_t simulcast_svc_idx);
brandtr8bc93852017-02-15 05:19:51 -0800163
Rasmus Brandt0f1c0bd2018-03-12 10:01:16 +0100164 // In order to supply the SVC decoders with super frames containing all
165 // lower layer frames, we merge and store the layer frames in this method.
Sergey Silkin645e2e02018-04-06 09:42:13 +0200166 const webrtc::EncodedImage* BuildAndStoreSuperframe(
Rasmus Brandt0f1c0bd2018-03-12 10:01:16 +0100167 const EncodedImage& encoded_image,
168 const VideoCodecType codec,
169 size_t frame_number,
Sergey Silkin645e2e02018-04-06 09:42:13 +0200170 size_t simulcast_svc_idx,
171 bool inter_layer_predicted) RTC_RUN_ON(sequence_checker_);
Sergey Silkin10d9d592018-02-01 13:25:17 +0100172
Rasmus Brandtd062a3c2018-03-08 16:45:54 +0100173 // Test input/output.
Kári Tristan Helgason169005d2018-05-22 13:34:14 +0200174 VideoCodecTestFixture::Config config_ RTC_GUARDED_BY(sequence_checker_);
Sergey Silkin10d9d592018-02-01 13:25:17 +0100175 const size_t num_simulcast_or_spatial_layers_;
Kári Tristan Helgason169005d2018-05-22 13:34:14 +0200176 VideoCodecTestStats* const stats_;
Sergey Silkin10d9d592018-02-01 13:25:17 +0100177
Rasmus Brandtd062a3c2018-03-08 16:45:54 +0100178 // Codecs.
brandtr8bc93852017-02-15 05:19:51 -0800179 webrtc::VideoEncoder* const encoder_;
Sergey Silkin10d9d592018-02-01 13:25:17 +0100180 VideoDecoderList* const decoders_;
brandtraebc61e2017-02-28 07:13:47 -0800181 const std::unique_ptr<VideoBitrateAllocator> bitrate_allocator_;
Erik Språng566124a2018-04-23 12:32:22 +0200182 VideoBitrateAllocation bitrate_allocation_ RTC_GUARDED_BY(sequence_checker_);
Rasmus Brandtd062a3c2018-03-08 16:45:54 +0100183 uint32_t framerate_fps_ RTC_GUARDED_BY(sequence_checker_);
brandtraebc61e2017-02-28 07:13:47 -0800184
185 // Adapters for the codec callbacks.
brandtrbdd555c2017-08-21 01:34:04 -0700186 VideoProcessorEncodeCompleteCallback encode_callback_;
Sergey Silkin645e2e02018-04-06 09:42:13 +0200187 // Assign separate callback object to each decoder. This allows us to identify
188 // decoded layer in frame decode callback.
189 // simulcast_svc_idx -> decode callback.
190 std::vector<std::unique_ptr<VideoProcessorDecodeCompleteCallback>>
191 decode_callback_;
brandtraebc61e2017-02-28 07:13:47 -0800192
Rasmus Brandtd062a3c2018-03-08 16:45:54 +0100193 // Each call to ProcessFrame() will read one frame from |input_frame_reader_|.
194 FrameReader* const input_frame_reader_;
195
196 // Input frames are used as reference for frame quality evaluations.
Sergey Silkin64eaa992017-11-17 14:47:32 +0100197 // Async codecs might queue frames. To handle that we keep input frame
198 // and release it after corresponding coded frame is decoded and quality
199 // measurement is done.
Rasmus Brandtd062a3c2018-03-08 16:45:54 +0100200 // frame_number -> frame.
201 std::map<size_t, VideoFrame> input_frames_ RTC_GUARDED_BY(sequence_checker_);
Sergey Silkin10d9d592018-02-01 13:25:17 +0100202
203 // Encoder delivers coded frame layer-by-layer. We store coded frames and
204 // then, after all layers are encoded, decode them. Such separation of
205 // frame processing on superframe level simplifies encoding/decoding time
206 // measurement.
Rasmus Brandt0f1c0bd2018-03-12 10:01:16 +0100207 // simulcast_svc_idx -> merged SVC encoded frame.
208 std::vector<EncodedImage> merged_encoded_frames_
Sergey Silkin10d9d592018-02-01 13:25:17 +0100209 RTC_GUARDED_BY(sequence_checker_);
210
Rasmus Brandtd062a3c2018-03-08 16:45:54 +0100211 // These (optional) file writers are used to persistently store the encoded
212 // and decoded bitstreams. Each frame writer is enabled by being non-null.
213 IvfFileWriterList* const encoded_frame_writers_;
214 FrameWriterList* const decoded_frame_writers_;
brandtr8bc93852017-02-15 05:19:51 -0800215
Rasmus Brandt0f1c0bd2018-03-12 10:01:16 +0100216 // Metadata for inputed/encoded/decoded frames. Used for frame identification,
217 // frame drop detection, etc. We assume that encoded/decoded frames are
218 // ordered within each simulcast/spatial layer, but we do not make any
219 // assumptions of frame ordering between layers.
Rasmus Brandtd062a3c2018-03-08 16:45:54 +0100220 size_t last_inputed_frame_num_ RTC_GUARDED_BY(sequence_checker_);
221 size_t last_inputed_timestamp_ RTC_GUARDED_BY(sequence_checker_);
Rasmus Brandt0f1c0bd2018-03-12 10:01:16 +0100222 // simulcast_svc_idx -> encode status.
223 std::vector<bool> first_encoded_frame_ RTC_GUARDED_BY(sequence_checker_);
224 // simulcast_svc_idx -> frame_number.
225 std::vector<size_t> last_encoded_frame_num_ RTC_GUARDED_BY(sequence_checker_);
226 // simulcast_svc_idx -> decode status.
227 std::vector<bool> first_decoded_frame_ RTC_GUARDED_BY(sequence_checker_);
228 // simulcast_svc_idx -> frame_number.
229 std::vector<size_t> last_decoded_frame_num_ RTC_GUARDED_BY(sequence_checker_);
Sergey Silkin56138792018-05-02 10:50:55 +0200230 // simulcast_svc_idx -> buffer.
231 std::vector<rtc::Buffer> decoded_frame_buffer_
232 RTC_GUARDED_BY(sequence_checker_);
Rasmus Brandtd062a3c2018-03-08 16:45:54 +0100233
Sergey Silkinc89eed92018-04-01 23:57:51 +0200234 // Time spent in frame encode callback. It is accumulated for layers and
235 // reset when frame encode starts. When next layer is encoded post-encode time
236 // is substracted from measured encode time. Thus we get pure encode time.
237 int64_t post_encode_time_ns_ RTC_GUARDED_BY(sequence_checker_);
238
Rasmus Brandtd062a3c2018-03-08 16:45:54 +0100239 // This class must be operated on a TaskQueue.
brandtrc8c59052017-08-21 06:44:16 -0700240 rtc::SequencedTaskChecker sequence_checker_;
brandtrbdd555c2017-08-21 01:34:04 -0700241
242 RTC_DISALLOW_COPY_AND_ASSIGN(VideoProcessor);
kjellander@webrtc.org35a17562011-10-06 06:44:54 +0000243};
244
245} // namespace test
246} // namespace webrtc
247
Mirko Bonadei92ea95e2017-09-15 06:47:31 +0200248#endif // MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_H_