pbos@webrtc.org | 9115cde | 2014-12-09 10:36:40 +0000 | [diff] [blame] | 1 | /* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved. |
Ilya Nikolaevskiy | bf35298 | 2017-10-02 10:08:25 +0200 | [diff] [blame] | 2 | * |
| 3 | * Use of this source code is governed by a BSD-style license |
| 4 | * that can be found in the LICENSE file in the root of the source |
| 5 | * tree. An additional intellectual property rights grant can be found |
| 6 | * in the file PATENTS. All contributing project authors may |
| 7 | * be found in the AUTHORS file in the root of the source tree. |
| 8 | */ |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 9 | #ifndef MODULES_VIDEO_CODING_CODECS_VP8_SCREENSHARE_LAYERS_H_ |
| 10 | #define MODULES_VIDEO_CODING_CODECS_VP8_SCREENSHARE_LAYERS_H_ |
pbos@webrtc.org | 9115cde | 2014-12-09 10:36:40 +0000 | [diff] [blame] | 11 | |
Erik Språng | 59021ba | 2018-10-03 11:05:16 +0200 | [diff] [blame] | 12 | #include <map> |
| 13 | #include <memory> |
philipel | b19f27a | 2019-03-28 16:09:52 +0100 | [diff] [blame] | 14 | #include <utility> |
Erik Språng | 08127a9 | 2016-11-16 16:41:30 +0100 | [diff] [blame] | 15 | #include <vector> |
pbos@webrtc.org | e728ee0 | 2014-12-17 13:43:55 +0000 | [diff] [blame] | 16 | |
Elad Alon | 411b49b | 2019-01-29 14:05:55 +0100 | [diff] [blame] | 17 | #include "api/video_codecs/vp8_frame_config.h" |
Erik Språng | 4529fbc | 2018-10-12 10:30:31 +0200 | [diff] [blame] | 18 | #include "api/video_codecs/vp8_temporal_layers.h" |
Erik Språng | 59021ba | 2018-10-03 11:05:16 +0200 | [diff] [blame] | 19 | #include "modules/video_coding/codecs/vp8/include/temporal_layers_checker.h" |
philipel | 9df3353 | 2019-03-04 16:37:50 +0100 | [diff] [blame] | 20 | #include "modules/video_coding/include/video_codec_interface.h" |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 21 | #include "modules/video_coding/utility/frame_dropper.h" |
| 22 | #include "rtc_base/rate_statistics.h" |
Steve Anton | 10542f2 | 2019-01-11 09:11:00 -0800 | [diff] [blame] | 23 | #include "rtc_base/time_utils.h" |
pbos@webrtc.org | 9115cde | 2014-12-09 10:36:40 +0000 | [diff] [blame] | 24 | |
pbos@webrtc.org | 9115cde | 2014-12-09 10:36:40 +0000 | [diff] [blame] | 25 | namespace webrtc { |
| 26 | |
| 27 | struct CodecSpecificInfoVP8; |
sprang | b0fdfea | 2016-03-01 05:51:16 -0800 | [diff] [blame] | 28 | class Clock; |
pbos@webrtc.org | 9115cde | 2014-12-09 10:36:40 +0000 | [diff] [blame] | 29 | |
Elad Alon | cde8ab2 | 2019-03-20 11:56:20 +0100 | [diff] [blame] | 30 | class ScreenshareLayers final : public Vp8FrameBufferController { |
pbos@webrtc.org | 9115cde | 2014-12-09 10:36:40 +0000 | [diff] [blame] | 31 | public: |
sprang@webrtc.org | 70f74f3 | 2014-12-17 10:57:10 +0000 | [diff] [blame] | 32 | static const double kMaxTL0FpsReduction; |
| 33 | static const double kAcceptableTargetOvershoot; |
sprang | afe1f74 | 2016-04-12 02:45:13 -0700 | [diff] [blame] | 34 | static const int kMaxFrameIntervalMs; |
sprang@webrtc.org | 70f74f3 | 2014-12-17 10:57:10 +0000 | [diff] [blame] | 35 | |
Sebastian Jansson | e64a688 | 2019-03-01 18:04:07 +0100 | [diff] [blame] | 36 | explicit ScreenshareLayers(int num_temporal_layers); |
Erik Språng | fb2a66a | 2018-09-10 10:48:01 +0200 | [diff] [blame] | 37 | ~ScreenshareLayers() override; |
| 38 | |
Elad Alon | fb08781 | 2019-05-02 23:25:34 +0200 | [diff] [blame] | 39 | void SetQpLimits(size_t stream_index, int min_qp, int max_qp) override; |
| 40 | |
Elad Alon | cde8ab2 | 2019-03-20 11:56:20 +0100 | [diff] [blame] | 41 | size_t StreamCount() const override; |
| 42 | |
| 43 | bool SupportsEncoderFrameDropping(size_t stream_index) const override; |
pbos@webrtc.org | 9115cde | 2014-12-09 10:36:40 +0000 | [diff] [blame] | 44 | |
| 45 | // Returns the recommended VP8 encode flags needed. May refresh the decoder |
| 46 | // and/or update the reference buffers. |
Elad Alon | 979c442 | 2019-04-17 12:53:08 +0200 | [diff] [blame] | 47 | Vp8FrameConfig NextFrameConfig(size_t stream_index, |
| 48 | uint32_t rtp_timestamp) override; |
pbos@webrtc.org | 9115cde | 2014-12-09 10:36:40 +0000 | [diff] [blame] | 49 | |
Erik Språng | bb60a3a | 2018-03-19 18:25:10 +0100 | [diff] [blame] | 50 | // New target bitrate, per temporal layer. |
Elad Alon | cde8ab2 | 2019-03-20 11:56:20 +0100 | [diff] [blame] | 51 | void OnRatesUpdated(size_t stream_index, |
| 52 | const std::vector<uint32_t>& bitrates_bps, |
Erik Språng | bb60a3a | 2018-03-19 18:25:10 +0100 | [diff] [blame] | 53 | int framerate_fps) override; |
Erik Språng | 08127a9 | 2016-11-16 16:41:30 +0100 | [diff] [blame] | 54 | |
Elad Alon | fb08781 | 2019-05-02 23:25:34 +0200 | [diff] [blame] | 55 | Vp8EncoderConfig UpdateConfiguration(size_t stream_index) override; |
pbos@webrtc.org | 9115cde | 2014-12-09 10:36:40 +0000 | [diff] [blame] | 56 | |
Elad Alon | cde8ab2 | 2019-03-20 11:56:20 +0100 | [diff] [blame] | 57 | void OnEncodeDone(size_t stream_index, |
| 58 | uint32_t rtp_timestamp, |
Erik Språng | 59021ba | 2018-10-03 11:05:16 +0200 | [diff] [blame] | 59 | size_t size_bytes, |
| 60 | bool is_keyframe, |
| 61 | int qp, |
philipel | 9df3353 | 2019-03-04 16:37:50 +0100 | [diff] [blame] | 62 | CodecSpecificInfo* info) override; |
pbos@webrtc.org | 9115cde | 2014-12-09 10:36:40 +0000 | [diff] [blame] | 63 | |
Elad Alon | 6796ec2 | 2019-04-15 10:07:50 +0200 | [diff] [blame] | 64 | void OnFrameDropped(size_t stream_index, uint32_t rtp_timestamp) override; |
| 65 | |
Elad Alon | cde8ab2 | 2019-03-20 11:56:20 +0100 | [diff] [blame] | 66 | void OnPacketLossRateUpdate(float packet_loss_rate) override; |
| 67 | |
| 68 | void OnRttUpdate(int64_t rtt_ms) override; |
| 69 | |
Elad Alon | 6c371ca | 2019-04-04 12:28:51 +0200 | [diff] [blame] | 70 | void OnLossNotification( |
Elad Alon | 123ee9b | 2019-04-17 12:48:06 +0200 | [diff] [blame] | 71 | const VideoEncoder::LossNotification& loss_notification) override; |
Elad Alon | 6c371ca | 2019-04-04 12:28:51 +0200 | [diff] [blame] | 72 | |
pbos@webrtc.org | 9115cde | 2014-12-09 10:36:40 +0000 | [diff] [blame] | 73 | private: |
sprang | 916170a | 2017-05-23 07:47:55 -0700 | [diff] [blame] | 74 | enum class TemporalLayerState : int { kDrop, kTl0, kTl1, kTl1Sync }; |
pbos | 51f083c | 2017-05-04 06:39:04 -0700 | [diff] [blame] | 75 | |
philipel | b19f27a | 2019-03-28 16:09:52 +0100 | [diff] [blame] | 76 | struct DependencyInfo { |
| 77 | DependencyInfo() = default; |
| 78 | DependencyInfo(absl::string_view indication_symbols, |
| 79 | Vp8FrameConfig frame_config) |
| 80 | : decode_target_indications( |
| 81 | GenericFrameInfo::DecodeTargetInfo(indication_symbols)), |
| 82 | frame_config(frame_config) {} |
| 83 | |
Danil Chapovalov | c2f5686 | 2019-06-27 12:11:03 +0200 | [diff] [blame^] | 84 | absl::InlinedVector<DecodeTargetIndication, 10> decode_target_indications; |
philipel | b19f27a | 2019-03-28 16:09:52 +0100 | [diff] [blame] | 85 | Vp8FrameConfig frame_config; |
| 86 | }; |
| 87 | |
Erik Språng | 2c4c914 | 2015-06-24 11:24:44 +0200 | [diff] [blame] | 88 | bool TimeToSync(int64_t timestamp) const; |
sprang | c7805db | 2016-11-25 08:09:43 -0800 | [diff] [blame] | 89 | uint32_t GetCodecTargetBitrateKbps() const; |
pbos@webrtc.org | 9115cde | 2014-12-09 10:36:40 +0000 | [diff] [blame] | 90 | |
Elad Alon | fb08781 | 2019-05-02 23:25:34 +0200 | [diff] [blame] | 91 | const int number_of_temporal_layers_; |
| 92 | |
| 93 | // TODO(eladalon/sprang): These should be made into const-int set in the ctor. |
| 94 | absl::optional<int> min_qp_; |
| 95 | absl::optional<int> max_qp_; |
| 96 | |
pbos@webrtc.org | 9115cde | 2014-12-09 10:36:40 +0000 | [diff] [blame] | 97 | int active_layer_; |
Erik Språng | 2c4c914 | 2015-06-24 11:24:44 +0200 | [diff] [blame] | 98 | int64_t last_timestamp_; |
pbos@webrtc.org | 9115cde | 2014-12-09 10:36:40 +0000 | [diff] [blame] | 99 | int64_t last_sync_timestamp_; |
sprang | afe1f74 | 2016-04-12 02:45:13 -0700 | [diff] [blame] | 100 | int64_t last_emitted_tl0_timestamp_; |
Erik Språng | 27a457d | 2018-01-12 11:00:21 +0100 | [diff] [blame] | 101 | int64_t last_frame_time_ms_; |
Erik Språng | 2c4c914 | 2015-06-24 11:24:44 +0200 | [diff] [blame] | 102 | rtc::TimestampWrapAroundHandler time_wrap_handler_; |
Erik Språng | 2c4c914 | 2015-06-24 11:24:44 +0200 | [diff] [blame] | 103 | uint32_t max_debt_bytes_; |
sprang | ac4a90d | 2016-12-28 05:58:07 -0800 | [diff] [blame] | 104 | |
philipel | b19f27a | 2019-03-28 16:09:52 +0100 | [diff] [blame] | 105 | std::map<uint32_t, DependencyInfo> pending_frame_configs_; |
Erik Språng | 59021ba | 2018-10-03 11:05:16 +0200 | [diff] [blame] | 106 | |
sprang | ac4a90d | 2016-12-28 05:58:07 -0800 | [diff] [blame] | 107 | // Configured max framerate. |
Danil Chapovalov | 0040b66 | 2018-06-18 10:48:16 +0200 | [diff] [blame] | 108 | absl::optional<uint32_t> target_framerate_; |
sprang | ac4a90d | 2016-12-28 05:58:07 -0800 | [diff] [blame] | 109 | // Incoming framerate from capturer. |
Danil Chapovalov | 0040b66 | 2018-06-18 10:48:16 +0200 | [diff] [blame] | 110 | absl::optional<uint32_t> capture_framerate_; |
philipel | 9df3353 | 2019-03-04 16:37:50 +0100 | [diff] [blame] | 111 | |
sprang | ac4a90d | 2016-12-28 05:58:07 -0800 | [diff] [blame] | 112 | // Tracks what framerate we actually encode, and drops frames on overshoot. |
| 113 | RateStatistics encode_framerate_; |
Erik Språng | 08127a9 | 2016-11-16 16:41:30 +0100 | [diff] [blame] | 114 | bool bitrate_updated_; |
Erik Språng | 2c4c914 | 2015-06-24 11:24:44 +0200 | [diff] [blame] | 115 | |
sprang | 429600d | 2017-01-26 06:12:26 -0800 | [diff] [blame] | 116 | static constexpr int kMaxNumTemporalLayers = 2; |
Erik Språng | 2c4c914 | 2015-06-24 11:24:44 +0200 | [diff] [blame] | 117 | struct TemporalLayer { |
| 118 | TemporalLayer() |
| 119 | : state(State::kNormal), |
| 120 | enhanced_max_qp(-1), |
| 121 | last_qp(-1), |
| 122 | debt_bytes_(0), |
| 123 | target_rate_kbps_(0) {} |
| 124 | |
| 125 | enum class State { |
| 126 | kNormal, |
| 127 | kDropped, |
| 128 | kReencoded, |
| 129 | kQualityBoost, |
Erik Språng | b75d6b8 | 2018-08-13 16:05:33 +0200 | [diff] [blame] | 130 | kKeyFrame |
Erik Språng | 2c4c914 | 2015-06-24 11:24:44 +0200 | [diff] [blame] | 131 | } state; |
| 132 | |
| 133 | int enhanced_max_qp; |
| 134 | int last_qp; |
| 135 | uint32_t debt_bytes_; |
| 136 | uint32_t target_rate_kbps_; |
| 137 | |
| 138 | void UpdateDebt(int64_t delta_ms); |
| 139 | } layers_[kMaxNumTemporalLayers]; |
sprang | b0fdfea | 2016-03-01 05:51:16 -0800 | [diff] [blame] | 140 | |
| 141 | void UpdateHistograms(); |
Danil Chapovalov | c2f5686 | 2019-06-27 12:11:03 +0200 | [diff] [blame^] | 142 | FrameDependencyStructure GetTemplateStructure(int num_layers) const; |
philipel | 9df3353 | 2019-03-04 16:37:50 +0100 | [diff] [blame] | 143 | |
sprang | b0fdfea | 2016-03-01 05:51:16 -0800 | [diff] [blame] | 144 | // Data for histogram statistics. |
| 145 | struct Stats { |
| 146 | int64_t first_frame_time_ms_ = -1; |
| 147 | int64_t num_tl0_frames_ = 0; |
| 148 | int64_t num_tl1_frames_ = 0; |
| 149 | int64_t num_dropped_frames_ = 0; |
| 150 | int64_t num_overshoots_ = 0; |
| 151 | int64_t tl0_qp_sum_ = 0; |
| 152 | int64_t tl1_qp_sum_ = 0; |
| 153 | int64_t tl0_target_bitrate_sum_ = 0; |
| 154 | int64_t tl1_target_bitrate_sum_ = 0; |
| 155 | } stats_; |
Erik Språng | 59021ba | 2018-10-03 11:05:16 +0200 | [diff] [blame] | 156 | |
Elad Alon | fb08781 | 2019-05-02 23:25:34 +0200 | [diff] [blame] | 157 | Vp8EncoderConfig encoder_config_; |
| 158 | |
Erik Språng | 59021ba | 2018-10-03 11:05:16 +0200 | [diff] [blame] | 159 | // Optional utility used to verify reference validity. |
| 160 | std::unique_ptr<TemporalLayersChecker> checker_; |
pbos@webrtc.org | 9115cde | 2014-12-09 10:36:40 +0000 | [diff] [blame] | 161 | }; |
| 162 | } // namespace webrtc |
| 163 | |
Mirko Bonadei | 92ea95e | 2017-09-15 06:47:31 +0200 | [diff] [blame] | 164 | #endif // MODULES_VIDEO_CODING_CODECS_VP8_SCREENSHARE_LAYERS_H_ |