blob: 7ceb69970b8ec0b84cd7c3b92528bdd2b755a77c [file] [log] [blame]
marpan@webrtc.org5b883172014-11-01 06:10:48 +00001/*
2 * Copyright (c) 2014 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 *
10 */
11
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020012#ifndef MODULES_VIDEO_CODING_CODECS_VP9_VP9_IMPL_H_
13#define MODULES_VIDEO_CODING_CODECS_VP9_VP9_IMPL_H_
philipelcce46fc2015-12-21 03:04:49 -080014
Mirko Bonadei95adedb2018-11-19 09:52:37 +010015#ifdef RTC_ENABLE_VP9
16
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +020017#include <map>
kwiberg3f55dea2016-02-29 05:51:59 -080018#include <memory>
Ilya Nikolaevskiy61170682019-03-06 16:04:32 +010019#include <string>
philipelcce46fc2015-12-21 03:04:49 -080020#include <vector>
marpan@webrtc.org5b883172014-11-01 06:10:48 +000021
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020022#include "modules/video_coding/codecs/vp9/include/vp9.h"
Emircan Uysaler98badbc2018-06-28 10:59:02 -070023
Elad Alon370f93a2019-06-11 14:57:57 +020024#include "api/video_codecs/video_encoder.h"
Emircan Uysaler98badbc2018-06-28 10:59:02 -070025#include "media/base/vp9_profile.h"
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020026#include "modules/video_coding/codecs/vp9/vp9_frame_buffer_pool.h"
Sergey Silkinae3144c2018-08-29 01:05:26 +020027#include "modules/video_coding/utility/framerate_controller.h"
marpan@webrtc.org5b883172014-11-01 06:10:48 +000028
johannkoenig8225c402017-01-26 13:23:44 -080029#include "vpx/vp8cx.h"
pbos@webrtc.orge728ee02014-12-17 13:43:55 +000030#include "vpx/vpx_decoder.h"
31#include "vpx/vpx_encoder.h"
marpan@webrtc.org5b883172014-11-01 06:10:48 +000032
33namespace webrtc {
34
35class VP9EncoderImpl : public VP9Encoder {
36 public:
Emircan Uysaler98badbc2018-06-28 10:59:02 -070037 explicit VP9EncoderImpl(const cricket::VideoCodec& codec);
marpan@webrtc.org5b883172014-11-01 06:10:48 +000038
39 virtual ~VP9EncoderImpl();
40
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +000041 int Release() override;
marpan@webrtc.org5b883172014-11-01 06:10:48 +000042
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +000043 int InitEncode(const VideoCodec* codec_settings,
Elad Alon370f93a2019-06-11 14:57:57 +020044 const Settings& settings) override;
marpan@webrtc.org5b883172014-11-01 06:10:48 +000045
Miguel Casas-Sanchez47650702015-05-29 17:21:40 -070046 int Encode(const VideoFrame& input_image,
Niels Möller87e2d782019-03-07 10:18:23 +010047 const std::vector<VideoFrameType>* frame_types) override;
marpan@webrtc.org5b883172014-11-01 06:10:48 +000048
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +000049 int RegisterEncodeCompleteCallback(EncodedImageCallback* callback) override;
marpan@webrtc.org5b883172014-11-01 06:10:48 +000050
Erik Språng16cb8f52019-04-12 13:59:09 +020051 void SetRates(const RateControlParameters& parameters) override;
marpan@webrtc.org5b883172014-11-01 06:10:48 +000052
Erik Språng727d1642018-11-07 16:54:15 +010053 EncoderInfo GetEncoderInfo() const override;
Peter Boströmb7d9a972015-12-18 16:01:11 +010054
marpan@webrtc.org5b883172014-11-01 06:10:48 +000055 private:
marpan@webrtc.org38d11b82015-01-26 15:21:36 +000056 // Determine number of encoder threads to use.
57 int NumberOfThreads(int width, int height, int number_of_cores);
58
marpan@webrtc.org5b883172014-11-01 06:10:48 +000059 // Call encoder initialize function and set control settings.
60 int InitAndSetControlSettings(const VideoCodec* inst);
61
62 void PopulateCodecSpecific(CodecSpecificInfo* codec_specific,
Niels Möllerd3b8c632018-08-27 15:33:42 +020063 absl::optional<int>* spatial_idx,
marpan@webrtc.org5b883172014-11-01 06:10:48 +000064 const vpx_codec_cx_pkt& pkt,
Sergey Silkin88ce4ef2018-11-23 13:59:24 +010065 uint32_t timestamp);
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +020066 void FillReferenceIndices(const vpx_codec_cx_pkt& pkt,
67 const size_t pic_num,
68 const bool inter_layer_predicted,
69 CodecSpecificInfoVP9* vp9_info);
70 void UpdateReferenceBuffers(const vpx_codec_cx_pkt& pkt,
71 const size_t pic_num);
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +010072 vpx_svc_ref_frame_config_t SetReferences(
73 bool is_key_pic,
74 size_t first_active_spatial_layer_id);
marpan@webrtc.org5b883172014-11-01 06:10:48 +000075
sprangce4aef12015-11-02 07:23:20 -080076 bool ExplicitlyConfiguredSpatialLayers() const;
Erik Språng566124a2018-04-23 12:32:22 +020077 bool SetSvcRates(const VideoBitrateAllocation& bitrate_allocation);
asaperssona9455ab2015-07-31 06:10:09 -070078
79 virtual int GetEncodedLayerFrame(const vpx_codec_cx_pkt* pkt);
80
81 // Callback function for outputting packets per spatial layer.
82 static void EncoderOutputCodedPacketCallback(vpx_codec_cx_pkt* pkt,
83 void* user_data);
marpan@webrtc.org5b883172014-11-01 06:10:48 +000084
Sergey Silkinbc0f0d32018-04-24 21:29:14 +020085 void DeliverBufferedFrame(bool end_of_picture);
Sergey Silkin2a1f1832018-04-04 11:45:41 +020086
Sergey Silkinae3144c2018-08-29 01:05:26 +020087 bool DropFrame(uint8_t spatial_idx, uint32_t rtp_timestamp);
Sergey Silkinbe71a1e2018-05-17 16:46:43 +020088
marpan@webrtc.org5b883172014-11-01 06:10:48 +000089 // Determine maximum target for Intra frames
90 //
91 // Input:
92 // - optimal_buffer_size : Optimal buffer size
93 // Return Value : Max target size for Intra frames represented as
94 // percentage of the per frame bandwidth
95 uint32_t MaxIntraTarget(uint32_t optimal_buffer_size);
96
Ilya Nikolaevskiy61170682019-03-06 16:04:32 +010097 size_t SteadyStateSize(int sid, int tid);
98
marpan@webrtc.org5b883172014-11-01 06:10:48 +000099 EncodedImage encoded_image_;
Sergey Silkin2a1f1832018-04-04 11:45:41 +0200100 CodecSpecificInfo codec_specific_;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000101 EncodedImageCallback* encoded_complete_callback_;
102 VideoCodec codec_;
Emircan Uysaler98badbc2018-06-28 10:59:02 -0700103 const VP9Profile profile_;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000104 bool inited_;
105 int64_t timestamp_;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000106 int cpu_speed_;
107 uint32_t rc_max_intra_target_;
108 vpx_codec_ctx_t* encoder_;
109 vpx_codec_enc_cfg_t* config_;
110 vpx_image_t* raw_;
johannkoenig8225c402017-01-26 13:23:44 -0800111 vpx_svc_extra_cfg_t svc_params_;
asaperssona9455ab2015-07-31 06:10:09 -0700112 const VideoFrame* input_image_;
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200113 GofInfoVP9 gof_; // Contains each frame's temporal information for
114 // non-flexible mode.
Sergey Silkinbd0954e2018-05-03 14:14:09 +0200115 bool force_key_frame_;
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200116 size_t pics_since_key_;
asaperssona9455ab2015-07-31 06:10:09 -0700117 uint8_t num_temporal_layers_;
“Michael23c5a992018-06-21 11:07:21 -0500118 uint8_t num_spatial_layers_; // Number of configured SLs
119 uint8_t num_active_spatial_layers_; // Number of actively encoded SLs
Sergey Silkine7ce8882018-10-03 18:04:57 +0200120 bool layer_deactivation_requires_key_frame_;
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200121 bool is_svc_;
Sergey Silkin6a8f30e2018-04-26 11:03:49 +0200122 InterLayerPredMode inter_layer_pred_;
Sergey Silkin390f3582018-10-01 17:13:50 +0200123 bool external_ref_control_;
Erik Språngd3438aa2018-11-08 16:56:43 +0100124 const bool trusted_rate_controller_;
Erik Språng7a3fe892019-04-15 12:22:55 +0200125 const bool dynamic_rate_settings_;
Sergey Silkin88ce4ef2018-11-23 13:59:24 +0100126 const bool full_superframe_drop_;
Ilya Nikolaevskiy039a7142019-05-24 16:50:00 +0200127 bool dropping_only_base_layer_;
128 vpx_svc_frame_drop_t svc_drop_frame_;
Sergey Silkin88ce4ef2018-11-23 13:59:24 +0100129 bool first_frame_in_picture_;
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100130 VideoBitrateAllocation current_bitrate_allocation_;
Erik Språng7a3fe892019-04-15 12:22:55 +0200131 absl::optional<RateControlParameters> requested_rate_settings_;
Ilya Nikolaevskiy5546aef2018-12-04 15:54:52 +0100132 bool ss_info_needed_;
philipelcfc319b2015-11-10 07:17:23 -0800133
Sergey Silkin96f2c972018-09-05 21:07:17 +0200134 std::vector<FramerateController> framerate_controller_;
Sergey Silkind902d582018-05-18 17:31:19 +0200135
philipelcfc319b2015-11-10 07:17:23 -0800136 // Used for flexible mode.
137 bool is_flexible_mode_;
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200138 struct RefFrameBuffer {
139 RefFrameBuffer(size_t pic_num,
140 size_t spatial_layer_id,
141 size_t temporal_layer_id)
142 : pic_num(pic_num),
143 spatial_layer_id(spatial_layer_id),
144 temporal_layer_id(temporal_layer_id) {}
145 RefFrameBuffer() {}
Sergey Silkin82276592018-08-27 14:54:20 +0200146
147 bool operator==(const RefFrameBuffer& o) {
148 return pic_num == o.pic_num && spatial_layer_id == o.spatial_layer_id &&
149 temporal_layer_id == o.temporal_layer_id;
150 }
151
Sergey Silkin4e6cd5e2018-05-28 12:26:36 +0200152 size_t pic_num = 0;
153 size_t spatial_layer_id = 0;
154 size_t temporal_layer_id = 0;
155 };
156 std::map<size_t, RefFrameBuffer> ref_buf_;
Ilya Nikolaevskiy61170682019-03-06 16:04:32 +0100157
158 // Variable frame-rate related fields and methods.
159 const struct VariableFramerateExperiment {
160 bool enabled;
161 // Framerate is limited to this value in steady state.
162 float framerate_limit;
163 // This qp or below is considered a steady state.
164 int steady_state_qp;
165 // Frames of at least this percentage below ideal for configured bitrate are
166 // considered in a steady state.
167 int steady_state_undershoot_percentage;
168 // Number of consecutive frames with good QP and size required to detect
169 // the steady state.
170 int frames_before_steady_state;
171 } variable_framerate_experiment_;
172 static VariableFramerateExperiment ParseVariableFramerateConfig(
173 std::string group_name);
174 FramerateController variable_framerate_controller_;
175 int num_steady_state_frames_;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000176};
177
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000178class VP9DecoderImpl : public VP9Decoder {
179 public:
180 VP9DecoderImpl();
181
182 virtual ~VP9DecoderImpl();
183
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000184 int InitDecode(const VideoCodec* inst, int number_of_cores) override;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000185
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000186 int Decode(const EncodedImage& input_image,
187 bool missing_frames,
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000188 int64_t /*render_time_ms*/) override;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000189
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000190 int RegisterDecodeCompleteCallback(DecodedImageCallback* callback) override;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000191
kjellander@webrtc.org14665ff2015-03-04 12:58:35 +0000192 int Release() override;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000193
Peter Boströmb7d9a972015-12-18 16:01:11 +0100194 const char* ImplementationName() const override;
195
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000196 private:
Ilya Nikolaevskiyab62b2e2019-06-05 15:55:51 +0200197 int ReturnFrame(const vpx_image_t* img, uint32_t timestamp, int qp);
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000198
Henrik Boström9695d852015-05-06 10:42:15 +0200199 // Memory pool used to share buffers between libvpx and webrtc.
200 Vp9FrameBufferPool frame_buffer_pool_;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000201 DecodedImageCallback* decode_complete_callback_;
202 bool inited_;
pbos@webrtc.orge728ee02014-12-17 13:43:55 +0000203 vpx_codec_ctx_t* decoder_;
marpan@webrtc.org5b883172014-11-01 06:10:48 +0000204 bool key_frame_required_;
205};
206} // namespace webrtc
207
Mirko Bonadei95adedb2018-11-19 09:52:37 +0100208#endif // RTC_ENABLE_VP9
209
Mirko Bonadei92ea95e2017-09-15 06:47:31 +0200210#endif // MODULES_VIDEO_CODING_CODECS_VP9_VP9_IMPL_H_