blob: 1f4ade201e267c5a2c7f290bf71eae00ba591805 [file] [log] [blame]
niklase@google.com470e71d2011-07-07 08:21:25 +00001/*
pwestin@webrtc.org52fd98d2012-02-13 09:03:53 +00002 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
niklase@google.com470e71d2011-07-07 08:21:25 +00003 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#ifndef MODULES_VIDEO_CODING_GENERIC_ENCODER_H_
12#define MODULES_VIDEO_CODING_GENERIC_ENCODER_H_
niklase@google.com470e71d2011-07-07 08:21:25 +000013
philipel9d3ab612015-12-21 04:12:39 -080014#include <stdio.h>
Ilya Nikolaevskiyd79314f2017-10-23 10:45:37 +020015#include <list>
philipel9d3ab612015-12-21 04:12:39 -080016#include <vector>
17
Erik Språngd96b2752018-12-13 11:23:27 +010018#include "api/units/data_rate.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020019#include "modules/video_coding/include/video_codec_interface.h"
20#include "modules/video_coding/include/video_coding_defines.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020021#include "rtc_base/criticalsection.h"
22#include "rtc_base/race_checker.h"
sprang@webrtc.org40709352013-11-26 11:41:59 +000023
andresp@webrtc.org1df9dc32014-01-09 08:01:57 +000024namespace webrtc {
sprang@webrtc.org40709352013-11-26 11:41:59 +000025
Niels Möller6613f8e2019-01-10 10:30:21 +010026namespace media_optimization {
27class MediaOptimization;
28} // namespace media_optimization
29
30struct EncoderParameters {
31 EncoderParameters() : total_bitrate(DataRate::Zero()), input_frame_rate(0) {}
32 EncoderParameters(DataRate total_bitrate,
33 const VideoBitrateAllocation& allocation,
34 uint32_t framerate)
35 : total_bitrate(total_bitrate),
36 target_bitrate(allocation),
37 input_frame_rate(framerate) {}
38
39 // Total bitrate allocated for this encoder.
40 DataRate total_bitrate;
41 // The bitrate allocation, across spatial and/or temporal layers. Note that
42 // the sum of these might be less than |total_bitrate| if the allocator has
43 // capped the bitrate for some configuration.
44 VideoBitrateAllocation target_bitrate;
45 // The input frame rate to the encoder in fps, measured in the video sender.
46 uint32_t input_frame_rate;
47};
48
philipel9d3ab612015-12-21 04:12:39 -080049class VCMEncodedFrameCallback : public EncodedImageCallback {
50 public:
Niels Möller6613f8e2019-01-10 10:30:21 +010051 VCMEncodedFrameCallback(EncodedImageCallback* post_encode_callback,
52 media_optimization::MediaOptimization* media_opt);
Stefan Holmerdbdb3a02018-07-17 16:03:46 +020053 ~VCMEncodedFrameCallback() override;
niklase@google.com470e71d2011-07-07 08:21:25 +000054
sprang3911c262016-04-15 01:24:14 -070055 // Implements EncodedImageCallback.
Sergey Ulanov525df3f2016-08-02 17:46:41 -070056 EncodedImageCallback::Result OnEncodedImage(
57 const EncodedImage& encoded_image,
58 const CodecSpecificInfo* codec_specific_info,
59 const RTPFragmentationHeader* fragmentation) override;
ilnik04f4d122017-06-19 07:18:55 -070060
sprang3911c262016-04-15 01:24:14 -070061 void SetInternalSource(bool internal_source) {
62 internal_source_ = internal_source;
63 }
guoweis@webrtc.org54d072e2015-03-17 21:54:50 +000064
ilnik04f4d122017-06-19 07:18:55 -070065 // Timing frames configuration methods. These 4 should be called before
66 // |OnEncodedImage| at least once.
67 void OnTargetBitrateChanged(size_t bitrate_bytes_per_sec,
68 size_t simulcast_svc_idx);
69
70 void OnFrameRateChanged(size_t framerate);
71
Ilya Nikolaevskiy76f2a852017-11-16 14:33:53 +010072 void OnEncodeStarted(uint32_t rtp_timestamps,
73 int64_t capture_time_ms,
74 size_t simulcast_svc_idx);
ilnik04f4d122017-06-19 07:18:55 -070075
76 void SetTimingFramesThresholds(
77 const VideoCodec::TimingFrameTriggerThresholds& thresholds) {
78 rtc::CritScope crit(&timing_params_lock_);
79 timing_frames_thresholds_ = thresholds;
80 }
81
Ilya Nikolaevskiyd79314f2017-10-23 10:45:37 +020082 // Clears all data stored by OnEncodeStarted().
83 void Reset() {
84 rtc::CritScope crit(&timing_params_lock_);
85 timing_frames_info_.clear();
86 last_timing_frame_time_ms_ = -1;
Ilya Nikolaevskiyb9fb78f2017-11-14 14:13:47 +010087 reordered_frames_logged_messages_ = 0;
88 stalled_encoder_logged_messages_ = 0;
Ilya Nikolaevskiyd79314f2017-10-23 10:45:37 +020089 }
90
philipel9d3ab612015-12-21 04:12:39 -080091 private:
Ilya Nikolaevskiy764aeb72018-04-03 10:01:52 +020092 // For non-internal-source encoders, returns encode started time and fixes
93 // capture timestamp for the frame, if corrupted by the encoder.
Danil Chapovalov0040b662018-06-18 10:48:16 +020094 absl::optional<int64_t> ExtractEncodeStartTime(size_t simulcast_svc_idx,
95 EncodedImage* encoded_image)
Ilya Nikolaevskiy764aeb72018-04-03 10:01:52 +020096 RTC_EXCLUSIVE_LOCKS_REQUIRED(timing_params_lock_);
97
98 void FillTimingInfo(size_t simulcast_svc_idx, EncodedImage* encoded_image);
99
ilnik04f4d122017-06-19 07:18:55 -0700100 rtc::CriticalSection timing_params_lock_;
sprang3911c262016-04-15 01:24:14 -0700101 bool internal_source_;
perkj376b1922016-05-02 11:35:24 -0700102 EncodedImageCallback* const post_encode_callback_;
Niels Möller6613f8e2019-01-10 10:30:21 +0100103 media_optimization::MediaOptimization* const media_opt_;
ilnik04f4d122017-06-19 07:18:55 -0700104
Ilya Nikolaevskiyd79314f2017-10-23 10:45:37 +0200105 struct EncodeStartTimeRecord {
Ilya Nikolaevskiy76f2a852017-11-16 14:33:53 +0100106 EncodeStartTimeRecord(uint32_t timestamp,
107 int64_t capture_time,
108 int64_t encode_start_time)
109 : rtp_timestamp(timestamp),
110 capture_time_ms(capture_time),
Ilya Nikolaevskiyd79314f2017-10-23 10:45:37 +0200111 encode_start_time_ms(encode_start_time) {}
Ilya Nikolaevskiy76f2a852017-11-16 14:33:53 +0100112 uint32_t rtp_timestamp;
Ilya Nikolaevskiyd79314f2017-10-23 10:45:37 +0200113 int64_t capture_time_ms;
114 int64_t encode_start_time_ms;
115 };
ilnik04f4d122017-06-19 07:18:55 -0700116 struct TimingFramesLayerInfo {
Stefan Holmerdbdb3a02018-07-17 16:03:46 +0200117 TimingFramesLayerInfo();
118 ~TimingFramesLayerInfo();
ilnik04f4d122017-06-19 07:18:55 -0700119 size_t target_bitrate_bytes_per_sec = 0;
Ilya Nikolaevskiyd79314f2017-10-23 10:45:37 +0200120 std::list<EncodeStartTimeRecord> encode_start_list;
ilnik04f4d122017-06-19 07:18:55 -0700121 };
122 // Separate instance for each simulcast stream or spatial layer.
123 std::vector<TimingFramesLayerInfo> timing_frames_info_
danilchap56359be2017-09-07 07:53:45 -0700124 RTC_GUARDED_BY(timing_params_lock_);
125 size_t framerate_ RTC_GUARDED_BY(timing_params_lock_);
126 int64_t last_timing_frame_time_ms_ RTC_GUARDED_BY(timing_params_lock_);
ilnik04f4d122017-06-19 07:18:55 -0700127 VideoCodec::TimingFrameTriggerThresholds timing_frames_thresholds_
danilchap56359be2017-09-07 07:53:45 -0700128 RTC_GUARDED_BY(timing_params_lock_);
Ilya Nikolaevskiy76f2a852017-11-16 14:33:53 +0100129 size_t incorrect_capture_time_logged_messages_
130 RTC_GUARDED_BY(timing_params_lock_);
Ilya Nikolaevskiyb9fb78f2017-11-14 14:13:47 +0100131 size_t reordered_frames_logged_messages_ RTC_GUARDED_BY(timing_params_lock_);
132 size_t stalled_encoder_logged_messages_ RTC_GUARDED_BY(timing_params_lock_);
ilnik6d5b4d62017-08-30 03:32:14 -0700133
134 // Experiment groups parsed from field trials for realtime video ([0]) and
135 // screenshare ([1]). 0 means no group specified. Positive values are
136 // experiment group numbers incremented by 1.
137 uint8_t experiment_groups_[2];
sprang3911c262016-04-15 01:24:14 -0700138};
sprang@webrtc.org40709352013-11-26 11:41:59 +0000139
philipel9d3ab612015-12-21 04:12:39 -0800140class VCMGenericEncoder {
141 friend class VCMCodecDataBase;
Peter Boström69ccb332015-10-29 16:30:23 +0100142
philipel9d3ab612015-12-21 04:12:39 -0800143 public:
144 VCMGenericEncoder(VideoEncoder* encoder,
philipel9d3ab612015-12-21 04:12:39 -0800145 VCMEncodedFrameCallback* encoded_frame_callback,
sprang3911c262016-04-15 01:24:14 -0700146 bool internal_source);
philipel9d3ab612015-12-21 04:12:39 -0800147 ~VCMGenericEncoder();
philipel9d3ab612015-12-21 04:12:39 -0800148 int32_t Release();
philipel9d3ab612015-12-21 04:12:39 -0800149 int32_t InitEncode(const VideoCodec* settings,
sprang3911c262016-04-15 01:24:14 -0700150 int32_t number_of_cores,
151 size_t max_payload_size);
152 int32_t Encode(const VideoFrame& frame,
153 const CodecSpecificInfo* codec_specific,
154 const std::vector<FrameType>& frame_types);
niklase@google.com470e71d2011-07-07 08:21:25 +0000155
Niels Möller6613f8e2019-01-10 10:30:21 +0100156 void SetEncoderParameters(const EncoderParameters& params);
157 EncoderParameters GetEncoderParameters() const;
niklase@google.com470e71d2011-07-07 08:21:25 +0000158
philipel9d3ab612015-12-21 04:12:39 -0800159 int32_t RequestFrame(const std::vector<FrameType>& frame_types);
philipel9d3ab612015-12-21 04:12:39 -0800160 bool InternalSource() const;
Erik Språngd3438aa2018-11-08 16:56:43 +0100161 VideoEncoder::EncoderInfo GetEncoderInfo() const;
philipel9d3ab612015-12-21 04:12:39 -0800162
163 private:
Peter Boström02bafc62016-07-01 12:45:15 +0200164 rtc::RaceChecker race_checker_;
165
danilchap56359be2017-09-07 07:53:45 -0700166 VideoEncoder* const encoder_ RTC_GUARDED_BY(race_checker_);
philipel9d3ab612015-12-21 04:12:39 -0800167 VCMEncodedFrameCallback* const vcm_encoded_frame_callback_;
168 const bool internal_source_;
pbos5ad935c2016-01-25 03:52:44 -0800169 rtc::CriticalSection params_lock_;
Niels Möller6613f8e2019-01-10 10:30:21 +0100170 EncoderParameters encoder_params_ RTC_GUARDED_BY(params_lock_);
Ilya Nikolaevskiye0da9ea2017-11-08 14:39:02 +0100171 size_t streams_or_svc_num_ RTC_GUARDED_BY(race_checker_);
172 VideoCodecType codec_type_ RTC_GUARDED_BY(race_checker_);
sprang3911c262016-04-15 01:24:14 -0700173};
niklase@google.com470e71d2011-07-07 08:21:25 +0000174
pbos@webrtc.orgd900e8b2013-07-03 15:12:26 +0000175} // namespace webrtc
niklase@google.com470e71d2011-07-07 08:21:25 +0000176
Mirko Bonadei92ea95e2017-09-15 06:47:31 +0200177#endif // MODULES_VIDEO_CODING_GENERIC_ENCODER_H_