Reland "RtpEncodingParameters::request_resolution patch 1"
This reverts commit b625101da8d798c936cfd695505a5514644158b0.
Reason for revert: Found problem that was specific how
configuration is handled for VP9. A 1-line change in webrtc_video_engine.cc line 3715.
Thanks Rasmus and great that this was tested!
Original change's description:
> Revert "RtpEncodingParameters::request_resolution patch 1"
>
> This reverts commit ef7359e679e579ccb79afacf5c42e8c6020124e2.
>
> Reason for revert: Breaks downstream test
>
> Original change's description:
> > RtpEncodingParameters::request_resolution patch 1
> >
> > This patch adds RtpEncodingParameters::request_resolution
> > with documentation and plumming. No behaviour is changed yet.
> >
> > Bug: webrtc:14451
> > Change-Id: I1f4f83a312ee8c293e3d8f02b950751e62048304
> > Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/276262
> > Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
> > Reviewed-by: Henrik Boström <hbos@webrtc.org>
> > Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
> > Reviewed-by: Tomas Gunnarsson <tommi@webrtc.org>
> > Commit-Queue: Jonas Oreland <jonaso@webrtc.org>
> > Cr-Commit-Position: refs/heads/main@{#38172}
>
> Bug: webrtc:14451
> Change-Id: I4b9590e23ec38e9e1c2e51a4600ef96b129439f2
> No-Presubmit: true
> No-Tree-Checks: true
> No-Try: true
> Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/276541
> Commit-Queue: Björn Terelius <terelius@webrtc.org>
> Bot-Commit: rubber-stamper@appspot.gserviceaccount.com <rubber-stamper@appspot.gserviceaccount.com>
> Reviewed-by: Jonas Oreland <jonaso@webrtc.org>
> Owners-Override: Björn Terelius <terelius@webrtc.org>
> Cr-Commit-Position: refs/heads/main@{#38176}
Bug: webrtc:14451
Change-Id: Ica9b74180bce22d09bf289126bb5ac137bf9eb70
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/276543
Reviewed-by: Henrik Boström <hbos@webrtc.org>
Reviewed-by: Tomas Gunnarsson <tommi@webrtc.org>
Commit-Queue: Jonas Oreland <jonaso@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38178}
diff --git a/api/BUILD.gn b/api/BUILD.gn
index 9445e3c..5bf7396 100644
--- a/api/BUILD.gn
+++ b/api/BUILD.gn
@@ -454,6 +454,7 @@
"../rtc_base:checks",
"../rtc_base:stringutils",
"../rtc_base/system:rtc_export",
+ "video:resolution",
"video_codecs:scalability_mode",
]
absl_deps = [
diff --git a/api/rtp_parameters.h b/api/rtp_parameters.h
index e311577..0d3c9df 100644
--- a/api/rtp_parameters.h
+++ b/api/rtp_parameters.h
@@ -23,6 +23,7 @@
#include "api/media_types.h"
#include "api/priority.h"
#include "api/rtp_transceiver_direction.h"
+#include "api/video/resolution.h"
#include "api/video_codecs/scalability_mode.h"
#include "rtc_base/system/rtc_export.h"
@@ -502,6 +503,24 @@
// https://w3c.github.io/webrtc-svc/#rtcrtpencodingparameters
absl::optional<std::string> scalability_mode;
+ // Requested encode resolution.
+ //
+ // This field provides an alternative to `scale_resolution_down_by`
+ // that is not dependent on the video source.
+ //
+ // When setting requested_resolution it is not necessary to adapt the
+ // video source using OnOutputFormatRequest, since the VideoStreamEncoder
+ // will apply downscaling if necessary. requested_resolution will also be
+ // propagated to the video source, this allows downscaling earlier in the
+ // pipeline which can be beneficial if the source is consumed by multiple
+ // encoders, but is not strictly necessary.
+ //
+ // The `requested_resolution` is subject to resource adaptation.
+ //
+ // It is an error to set both `requested_resolution` and
+ // `scale_resolution_down_by`.
+ absl::optional<Resolution> requested_resolution;
+
// For an RtpSender, set to true to cause this encoding to be encoded and
// sent, and false for it not to be encoded and sent. This allows control
// across multiple encodings of a sender for turning simulcast layers on and
@@ -527,7 +546,8 @@
num_temporal_layers == o.num_temporal_layers &&
scale_resolution_down_by == o.scale_resolution_down_by &&
active == o.active && rid == o.rid &&
- adaptive_ptime == o.adaptive_ptime;
+ adaptive_ptime == o.adaptive_ptime &&
+ requested_resolution == o.requested_resolution;
}
bool operator!=(const RtpEncodingParameters& o) const {
return !(*this == o);
diff --git a/api/video/BUILD.gn b/api/video/BUILD.gn
index 060b9e4..ee62abd 100644
--- a/api/video/BUILD.gn
+++ b/api/video/BUILD.gn
@@ -131,6 +131,11 @@
public = [ "render_resolution.h" ]
}
+rtc_source_set("resolution") {
+ visibility = [ "*" ]
+ public = [ "resolution.h" ]
+}
+
rtc_library("encoded_image") {
visibility = [ "*" ]
sources = [
diff --git a/api/video/recordable_encoded_frame.h b/api/video/recordable_encoded_frame.h
index 702f4d7..47ea23f 100644
--- a/api/video/recordable_encoded_frame.h
+++ b/api/video/recordable_encoded_frame.h
@@ -24,6 +24,7 @@
class RecordableEncodedFrame {
public:
// Encoded resolution in pixels
+ // TODO(bugs.webrtc.org/12114) : remove in favor of Resolution.
struct EncodedResolution {
bool empty() const { return width == 0 && height == 0; }
diff --git a/api/video/render_resolution.h b/api/video/render_resolution.h
index edcf8f8..fcf4f12 100644
--- a/api/video/render_resolution.h
+++ b/api/video/render_resolution.h
@@ -13,6 +13,7 @@
namespace webrtc {
+// TODO(bugs.webrtc.org/12114) : remove in favor of Resolution.
class RenderResolution {
public:
constexpr RenderResolution() = default;
diff --git a/api/video/resolution.h b/api/video/resolution.h
new file mode 100644
index 0000000..99cb622
--- /dev/null
+++ b/api/video/resolution.h
@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef API_VIDEO_RESOLUTION_H_
+#define API_VIDEO_RESOLUTION_H_
+
+namespace webrtc {
+
+// A struct representing a video resolution in pixels.
+struct Resolution {
+ int width = 0;
+ int height = 0;
+};
+
+inline bool operator==(const Resolution& lhs, const Resolution& rhs) {
+ return lhs.width == rhs.width && lhs.height == rhs.height;
+}
+
+inline bool operator!=(const Resolution& lhs, const Resolution& rhs) {
+ return !(lhs == rhs);
+}
+
+} // namespace webrtc
+
+#endif // API_VIDEO_RESOLUTION_H_
diff --git a/api/video/video_source_interface.h b/api/video/video_source_interface.h
index 5eb4ebf..72937c7 100644
--- a/api/video/video_source_interface.h
+++ b/api/video/video_source_interface.h
@@ -80,6 +80,24 @@
// Note that the `resolutions` can change while frames are in flight and
// should only be used as a hint when constructing the webrtc::VideoFrame.
std::vector<FrameSize> resolutions;
+
+ // This is the resolution requested by the user using RtpEncodingParameters.
+ absl::optional<FrameSize> requested_resolution;
+
+ // `active` : is (any) of the layers/sink(s) active.
+ bool is_active = false;
+
+ // This sub-struct contains information computed by VideoBroadcaster
+ // that aggregates several VideoSinkWants (and sends them to
+ // AdaptedVideoTrackSource).
+ struct Aggregates {
+ // `active_without_requested_resolution` is set by VideoBroadcaster
+ // when aggregating sink wants if there exists any sink (encoder) that is
+ // active but has not set the `requested_resolution`, i.e is relying on
+ // OnOutputFormatRequest to handle encode resolution.
+ bool any_active_without_requested_resolution = false;
+ };
+ absl::optional<Aggregates> aggregates;
};
inline bool operator==(const VideoSinkWants::FrameSize& a,
@@ -87,6 +105,11 @@
return a.width == b.width && a.height == b.height;
}
+inline bool operator!=(const VideoSinkWants::FrameSize& a,
+ const VideoSinkWants::FrameSize& b) {
+ return !(a == b);
+}
+
template <typename VideoFrameT>
class VideoSourceInterface {
public:
diff --git a/api/video_codecs/BUILD.gn b/api/video_codecs/BUILD.gn
index 3f933b9..d6b7392 100644
--- a/api/video_codecs/BUILD.gn
+++ b/api/video_codecs/BUILD.gn
@@ -84,6 +84,7 @@
"../units:data_rate",
"../video:encoded_image",
"../video:render_resolution",
+ "../video:resolution",
"../video:video_bitrate_allocation",
"../video:video_codec_constants",
"../video:video_frame",
diff --git a/api/video_codecs/video_encoder_config.h b/api/video_codecs/video_encoder_config.h
index 86d89d5..3d1b176 100644
--- a/api/video_codecs/video_encoder_config.h
+++ b/api/video_codecs/video_encoder_config.h
@@ -18,6 +18,7 @@
#include "absl/types/optional.h"
#include "api/scoped_refptr.h"
+#include "api/video/resolution.h"
#include "api/video_codecs/scalability_mode.h"
#include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_codec.h"
@@ -32,10 +33,11 @@
VideoStream(const VideoStream& other);
std::string ToString() const;
- // Width in pixels.
+ // Width/Height in pixels.
+ // This is the actual width and height used to configure encoder,
+ // which might be less than `requested_resolution` due to adaptation
+ // or due to the source providing smaller frames than requested.
size_t width;
-
- // Height in pixels.
size_t height;
// Frame rate in fps.
@@ -69,6 +71,17 @@
// If this stream is enabled by the user, or not.
bool active;
+
+ // An optional user supplied max_frame_resolution
+ // than can be set independently of (adapted) VideoSource.
+ // This value is set from RtpEncodingParameters::requested_resolution
+ // (i.e. used for signaling app-level settings).
+ //
+ // The actual encode resolution is in `width` and `height`,
+ // which can be lower than requested_resolution,
+ // e.g. if source only provides lower resolution or
+ // if resource adaptation is active.
+ absl::optional<Resolution> requested_resolution;
};
class VideoEncoderConfig {