Sami Kalliomäki | e2410e9 | 2017-06-02 14:46:12 +0200 | [diff] [blame] | 1 | /* |
| 2 | * Copyright 2017 The WebRTC project authors. All Rights Reserved. |
| 3 | * |
| 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
| 9 | */ |
| 10 | |
| 11 | package org.webrtc; |
| 12 | |
Artem Titarenko | 69540f4 | 2018-12-10 12:30:46 +0100 | [diff] [blame] | 13 | import android.support.annotation.Nullable; |
Sergey Silkin | be0adee | 2019-06-26 14:11:03 +0200 | [diff] [blame] | 14 | import java.util.Collections; |
| 15 | import java.util.List; |
Magnus Jedvert | 1f2a3e7 | 2017-11-23 16:56:44 +0100 | [diff] [blame] | 16 | import org.webrtc.EncodedImage; |
| 17 | |
Sami Kalliomäki | e2410e9 | 2017-06-02 14:46:12 +0200 | [diff] [blame] | 18 | /** |
| 19 | * Interface for a video encoder that can be used with WebRTC. All calls will be made on the |
Sami Kalliomäki | 5f5fc68 | 2017-10-19 11:34:08 +0200 | [diff] [blame] | 20 | * encoding thread. The encoder may be constructed on a different thread and changing thread after |
| 21 | * calling release is allowed. |
Sami Kalliomäki | e2410e9 | 2017-06-02 14:46:12 +0200 | [diff] [blame] | 22 | */ |
| 23 | public interface VideoEncoder { |
| 24 | /** Settings passed to the encoder by WebRTC. */ |
| 25 | public class Settings { |
| 26 | public final int numberOfCores; |
Bjorn Mellem | 5c4eebb | 2017-06-12 09:21:03 -0700 | [diff] [blame] | 27 | public final int width; |
| 28 | public final int height; |
| 29 | public final int startBitrate; // Kilobits per second. |
| 30 | public final int maxFramerate; |
Rasmus Brandt | 85f20cb | 2018-08-23 13:52:45 +0200 | [diff] [blame] | 31 | public final int numberOfSimulcastStreams; |
sakal | 07a3bd7 | 2017-09-04 03:57:21 -0700 | [diff] [blame] | 32 | public final boolean automaticResizeOn; |
Elad Alon | 370f93a | 2019-06-11 14:57:57 +0200 | [diff] [blame] | 33 | public final Capabilities capabilities; |
| 34 | |
| 35 | // TODO(bugs.webrtc.org/10720): Remove. |
| 36 | @Deprecated |
| 37 | public Settings(int numberOfCores, int width, int height, int startBitrate, int maxFramerate, |
| 38 | int numberOfSimulcastStreams, boolean automaticResizeOn) { |
| 39 | this(numberOfCores, width, height, startBitrate, maxFramerate, numberOfSimulcastStreams, |
| 40 | automaticResizeOn, new VideoEncoder.Capabilities(false /* lossNotification */)); |
| 41 | } |
Sami Kalliomäki | e2410e9 | 2017-06-02 14:46:12 +0200 | [diff] [blame] | 42 | |
Magnus Jedvert | 1f2a3e7 | 2017-11-23 16:56:44 +0100 | [diff] [blame] | 43 | @CalledByNative("Settings") |
sakal | 07a3bd7 | 2017-09-04 03:57:21 -0700 | [diff] [blame] | 44 | public Settings(int numberOfCores, int width, int height, int startBitrate, int maxFramerate, |
Elad Alon | 370f93a | 2019-06-11 14:57:57 +0200 | [diff] [blame] | 45 | int numberOfSimulcastStreams, boolean automaticResizeOn, Capabilities capabilities) { |
Sami Kalliomäki | e2410e9 | 2017-06-02 14:46:12 +0200 | [diff] [blame] | 46 | this.numberOfCores = numberOfCores; |
Bjorn Mellem | 5c4eebb | 2017-06-12 09:21:03 -0700 | [diff] [blame] | 47 | this.width = width; |
| 48 | this.height = height; |
| 49 | this.startBitrate = startBitrate; |
| 50 | this.maxFramerate = maxFramerate; |
Rasmus Brandt | 85f20cb | 2018-08-23 13:52:45 +0200 | [diff] [blame] | 51 | this.numberOfSimulcastStreams = numberOfSimulcastStreams; |
sakal | 07a3bd7 | 2017-09-04 03:57:21 -0700 | [diff] [blame] | 52 | this.automaticResizeOn = automaticResizeOn; |
Elad Alon | 370f93a | 2019-06-11 14:57:57 +0200 | [diff] [blame] | 53 | this.capabilities = capabilities; |
| 54 | } |
| 55 | } |
| 56 | |
| 57 | /** Capabilities (loss notification, etc.) passed to the encoder by WebRTC. */ |
| 58 | public class Capabilities { |
| 59 | /** |
| 60 | * The remote side has support for the loss notification RTCP feedback message format, and will |
| 61 | * be sending these feedback messages if necessary. |
| 62 | */ |
| 63 | public final boolean lossNotification; |
| 64 | |
| 65 | @CalledByNative("Capabilities") |
| 66 | public Capabilities(boolean lossNotification) { |
| 67 | this.lossNotification = lossNotification; |
Sami Kalliomäki | e2410e9 | 2017-06-02 14:46:12 +0200 | [diff] [blame] | 68 | } |
| 69 | } |
| 70 | |
| 71 | /** Additional info for encoding. */ |
| 72 | public class EncodeInfo { |
| 73 | public final EncodedImage.FrameType[] frameTypes; |
| 74 | |
Magnus Jedvert | 1f2a3e7 | 2017-11-23 16:56:44 +0100 | [diff] [blame] | 75 | @CalledByNative("EncodeInfo") |
Sami Kalliomäki | e2410e9 | 2017-06-02 14:46:12 +0200 | [diff] [blame] | 76 | public EncodeInfo(EncodedImage.FrameType[] frameTypes) { |
| 77 | this.frameTypes = frameTypes; |
| 78 | } |
| 79 | } |
| 80 | |
| 81 | // TODO(sakal): Add values to these classes as necessary. |
| 82 | /** Codec specific information about the encoded frame. */ |
| 83 | public class CodecSpecificInfo {} |
| 84 | |
| 85 | public class CodecSpecificInfoVP8 extends CodecSpecificInfo {} |
| 86 | |
| 87 | public class CodecSpecificInfoVP9 extends CodecSpecificInfo {} |
| 88 | |
| 89 | public class CodecSpecificInfoH264 extends CodecSpecificInfo {} |
| 90 | |
| 91 | /** |
| 92 | * Represents bitrate allocated for an encoder to produce frames. Bitrate can be divided between |
| 93 | * spatial and temporal layers. |
| 94 | */ |
| 95 | public class BitrateAllocation { |
| 96 | // First index is the spatial layer and second the temporal layer. |
Bjorn Mellem | 5c4eebb | 2017-06-12 09:21:03 -0700 | [diff] [blame] | 97 | public final int[][] bitratesBbs; |
Sami Kalliomäki | e2410e9 | 2017-06-02 14:46:12 +0200 | [diff] [blame] | 98 | |
| 99 | /** |
| 100 | * Initializes the allocation with a two dimensional array of bitrates. The first index of the |
| 101 | * array is the spatial layer and the second index in the temporal layer. |
| 102 | */ |
Magnus Jedvert | 1f2a3e7 | 2017-11-23 16:56:44 +0100 | [diff] [blame] | 103 | @CalledByNative("BitrateAllocation") |
Bjorn Mellem | 5c4eebb | 2017-06-12 09:21:03 -0700 | [diff] [blame] | 104 | public BitrateAllocation(int[][] bitratesBbs) { |
Sami Kalliomäki | e2410e9 | 2017-06-02 14:46:12 +0200 | [diff] [blame] | 105 | this.bitratesBbs = bitratesBbs; |
| 106 | } |
| 107 | |
| 108 | /** |
| 109 | * Gets the total bitrate allocated for all layers. |
| 110 | */ |
Bjorn Mellem | 5c4eebb | 2017-06-12 09:21:03 -0700 | [diff] [blame] | 111 | public int getSum() { |
| 112 | int sum = 0; |
| 113 | for (int[] spatialLayer : bitratesBbs) { |
| 114 | for (int bitrate : spatialLayer) { |
Sami Kalliomäki | e2410e9 | 2017-06-02 14:46:12 +0200 | [diff] [blame] | 115 | sum += bitrate; |
| 116 | } |
| 117 | } |
| 118 | return sum; |
| 119 | } |
| 120 | } |
| 121 | |
| 122 | /** Settings for WebRTC quality based scaling. */ |
| 123 | public class ScalingSettings { |
| 124 | public final boolean on; |
Sami Kalliomäki | e7592d8 | 2018-03-22 13:32:44 +0100 | [diff] [blame] | 125 | @Nullable public final Integer low; |
| 126 | @Nullable public final Integer high; |
Sami Kalliomäki | e2410e9 | 2017-06-02 14:46:12 +0200 | [diff] [blame] | 127 | |
| 128 | /** |
Niels Möller | d0dd90b | 2018-02-08 10:15:34 +0100 | [diff] [blame] | 129 | * Settings to disable quality based scaling. |
| 130 | */ |
| 131 | public static final ScalingSettings OFF = new ScalingSettings(); |
| 132 | |
| 133 | /** |
| 134 | * Creates settings to enable quality based scaling. |
| 135 | * |
| 136 | * @param low Average QP at which to scale up the resolution. |
| 137 | * @param high Average QP at which to scale down the resolution. |
| 138 | */ |
| 139 | public ScalingSettings(int low, int high) { |
| 140 | this.on = true; |
| 141 | this.low = low; |
| 142 | this.high = high; |
| 143 | } |
| 144 | |
| 145 | private ScalingSettings() { |
| 146 | this.on = false; |
| 147 | this.low = null; |
| 148 | this.high = null; |
| 149 | } |
| 150 | |
| 151 | // TODO(bugs.webrtc.org/8830): Below constructors are deprecated. |
| 152 | // Default thresholds are going away, so thresholds have to be set |
| 153 | // when scaling is on. |
| 154 | /** |
sakal | 07a3bd7 | 2017-09-04 03:57:21 -0700 | [diff] [blame] | 155 | * Creates quality based scaling setting. |
| 156 | * |
| 157 | * @param on True if quality scaling is turned on. |
| 158 | */ |
Niels Möller | d0dd90b | 2018-02-08 10:15:34 +0100 | [diff] [blame] | 159 | @Deprecated |
sakal | 07a3bd7 | 2017-09-04 03:57:21 -0700 | [diff] [blame] | 160 | public ScalingSettings(boolean on) { |
| 161 | this.on = on; |
| 162 | this.low = null; |
| 163 | this.high = null; |
| 164 | } |
| 165 | |
| 166 | /** |
| 167 | * Creates quality based scaling settings with custom thresholds. |
Sami Kalliomäki | e2410e9 | 2017-06-02 14:46:12 +0200 | [diff] [blame] | 168 | * |
| 169 | * @param on True if quality scaling is turned on. |
| 170 | * @param low Average QP at which to scale up the resolution. |
| 171 | * @param high Average QP at which to scale down the resolution. |
| 172 | */ |
Niels Möller | d0dd90b | 2018-02-08 10:15:34 +0100 | [diff] [blame] | 173 | @Deprecated |
Sami Kalliomäki | e2410e9 | 2017-06-02 14:46:12 +0200 | [diff] [blame] | 174 | public ScalingSettings(boolean on, int low, int high) { |
| 175 | this.on = on; |
| 176 | this.low = low; |
| 177 | this.high = high; |
| 178 | } |
Sami Kalliomäki | 8619e8a | 2018-04-17 14:44:36 +0200 | [diff] [blame] | 179 | |
| 180 | @Override |
| 181 | public String toString() { |
| 182 | return on ? "[ " + low + ", " + high + " ]" : "OFF"; |
| 183 | } |
Sami Kalliomäki | e2410e9 | 2017-06-02 14:46:12 +0200 | [diff] [blame] | 184 | } |
| 185 | |
Sergey Silkin | be0adee | 2019-06-26 14:11:03 +0200 | [diff] [blame] | 186 | /** |
Sergey Silkin | 3d642f8 | 2019-07-03 15:09:33 +0200 | [diff] [blame^] | 187 | * Bitrate limits for resolution. |
Sergey Silkin | be0adee | 2019-06-26 14:11:03 +0200 | [diff] [blame] | 188 | */ |
Sergey Silkin | 3d642f8 | 2019-07-03 15:09:33 +0200 | [diff] [blame^] | 189 | public class ResolutionBitrateLimits { |
Sergey Silkin | be0adee | 2019-06-26 14:11:03 +0200 | [diff] [blame] | 190 | /** |
Sergey Silkin | 3d642f8 | 2019-07-03 15:09:33 +0200 | [diff] [blame^] | 191 | * Maximum size of video frame, in pixels, the bitrate limits are intended for. |
Sergey Silkin | be0adee | 2019-06-26 14:11:03 +0200 | [diff] [blame] | 192 | */ |
| 193 | public final int frameSizePixels; |
| 194 | |
| 195 | /** |
| 196 | * Recommended minimum bitrate to start encoding. |
| 197 | */ |
| 198 | public final int minStartBitrateBps; |
| 199 | |
| 200 | /** |
| 201 | * Recommended minimum bitrate. |
| 202 | */ |
| 203 | public final int minBitrateBps; |
| 204 | |
| 205 | /** |
| 206 | * Recommended maximum bitrate. |
| 207 | */ |
| 208 | public final int maxBitrateBps; |
| 209 | |
Sergey Silkin | 3d642f8 | 2019-07-03 15:09:33 +0200 | [diff] [blame^] | 210 | public ResolutionBitrateLimits( |
Sergey Silkin | be0adee | 2019-06-26 14:11:03 +0200 | [diff] [blame] | 211 | int frameSizePixels, int minStartBitrateBps, int minBitrateBps, int maxBitrateBps) { |
| 212 | this.frameSizePixels = frameSizePixels; |
| 213 | this.minStartBitrateBps = minStartBitrateBps; |
| 214 | this.minBitrateBps = minBitrateBps; |
| 215 | this.maxBitrateBps = maxBitrateBps; |
| 216 | } |
| 217 | |
Sergey Silkin | 3d642f8 | 2019-07-03 15:09:33 +0200 | [diff] [blame^] | 218 | @CalledByNative("ResolutionBitrateLimits") |
Sergey Silkin | be0adee | 2019-06-26 14:11:03 +0200 | [diff] [blame] | 219 | public int getFrameSizePixels() { |
| 220 | return frameSizePixels; |
| 221 | } |
| 222 | |
Sergey Silkin | 3d642f8 | 2019-07-03 15:09:33 +0200 | [diff] [blame^] | 223 | @CalledByNative("ResolutionBitrateLimits") |
Sergey Silkin | be0adee | 2019-06-26 14:11:03 +0200 | [diff] [blame] | 224 | public int getMinStartBitrateBps() { |
| 225 | return minStartBitrateBps; |
| 226 | } |
| 227 | |
Sergey Silkin | 3d642f8 | 2019-07-03 15:09:33 +0200 | [diff] [blame^] | 228 | @CalledByNative("ResolutionBitrateLimits") |
Sergey Silkin | be0adee | 2019-06-26 14:11:03 +0200 | [diff] [blame] | 229 | public int getMinBitrateBps() { |
| 230 | return minBitrateBps; |
| 231 | } |
| 232 | |
Sergey Silkin | 3d642f8 | 2019-07-03 15:09:33 +0200 | [diff] [blame^] | 233 | @CalledByNative("ResolutionBitrateLimits") |
Sergey Silkin | be0adee | 2019-06-26 14:11:03 +0200 | [diff] [blame] | 234 | public int getMaxBitrateBps() { |
| 235 | return maxBitrateBps; |
| 236 | } |
| 237 | } |
| 238 | |
Sami Kalliomäki | e2410e9 | 2017-06-02 14:46:12 +0200 | [diff] [blame] | 239 | public interface Callback { |
Sami Kalliomäki | 11c51dd | 2018-02-07 12:50:47 +0100 | [diff] [blame] | 240 | /** |
| 241 | * Call to return an encoded frame. It is safe to assume the byte buffer held by |frame| is not |
| 242 | * accessed after the call to this method returns. |
| 243 | */ |
Sami Kalliomäki | e2410e9 | 2017-06-02 14:46:12 +0200 | [diff] [blame] | 244 | void onEncodedFrame(EncodedImage frame, CodecSpecificInfo info); |
| 245 | } |
| 246 | |
| 247 | /** |
Rasmus Brandt | 42a2fc9 | 2018-07-09 13:38:01 +0200 | [diff] [blame] | 248 | * The encoder implementation backing this interface is either 1) a Java |
| 249 | * encoder (e.g., an Android platform encoder), or alternatively 2) a native |
| 250 | * encoder (e.g., a software encoder or a C++ encoder adapter). |
| 251 | * |
| 252 | * For case 1), createNativeVideoEncoder() should return zero. |
| 253 | * In this case, we expect the native library to call the encoder through |
| 254 | * JNI using the Java interface declared below. |
| 255 | * |
| 256 | * For case 2), createNativeVideoEncoder() should return a non-zero value. |
| 257 | * In this case, we expect the native library to treat the returned value as |
| 258 | * a raw pointer of type webrtc::VideoEncoder* (ownership is transferred to |
| 259 | * the caller). The native library should then directly call the |
| 260 | * webrtc::VideoEncoder interface without going through JNI. All calls to |
| 261 | * the Java interface methods declared below should thus throw an |
| 262 | * UnsupportedOperationException. |
| 263 | */ |
| 264 | @CalledByNative |
| 265 | default long createNativeVideoEncoder() { |
| 266 | return 0; |
| 267 | } |
| 268 | |
| 269 | /** |
| 270 | * Returns true if the encoder is backed by hardware. |
| 271 | */ |
| 272 | @CalledByNative |
| 273 | default boolean isHardwareEncoder() { |
| 274 | return true; |
| 275 | } |
| 276 | |
| 277 | /** |
Sami Kalliomäki | e2410e9 | 2017-06-02 14:46:12 +0200 | [diff] [blame] | 278 | * Initializes the encoding process. Call before any calls to encode. |
| 279 | */ |
Magnus Jedvert | 56231d0 | 2017-10-31 17:47:06 +0100 | [diff] [blame] | 280 | @CalledByNative VideoCodecStatus initEncode(Settings settings, Callback encodeCallback); |
| 281 | |
Sami Kalliomäki | e2410e9 | 2017-06-02 14:46:12 +0200 | [diff] [blame] | 282 | /** |
| 283 | * Releases the encoder. No more calls to encode will be made after this call. |
| 284 | */ |
Magnus Jedvert | 56231d0 | 2017-10-31 17:47:06 +0100 | [diff] [blame] | 285 | @CalledByNative VideoCodecStatus release(); |
| 286 | |
Sami Kalliomäki | e2410e9 | 2017-06-02 14:46:12 +0200 | [diff] [blame] | 287 | /** |
| 288 | * Requests the encoder to encode a frame. |
| 289 | */ |
Magnus Jedvert | 56231d0 | 2017-10-31 17:47:06 +0100 | [diff] [blame] | 290 | @CalledByNative VideoCodecStatus encode(VideoFrame frame, EncodeInfo info); |
| 291 | |
Sami Kalliomäki | e2410e9 | 2017-06-02 14:46:12 +0200 | [diff] [blame] | 292 | /** Sets the bitrate allocation and the target framerate for the encoder. */ |
Magnus Jedvert | 56231d0 | 2017-10-31 17:47:06 +0100 | [diff] [blame] | 293 | @CalledByNative VideoCodecStatus setRateAllocation(BitrateAllocation allocation, int framerate); |
| 294 | |
Sami Kalliomäki | e2410e9 | 2017-06-02 14:46:12 +0200 | [diff] [blame] | 295 | /** Any encoder that wants to use WebRTC provided quality scaler must implement this method. */ |
Magnus Jedvert | 56231d0 | 2017-10-31 17:47:06 +0100 | [diff] [blame] | 296 | @CalledByNative ScalingSettings getScalingSettings(); |
| 297 | |
Sergey Silkin | 3d642f8 | 2019-07-03 15:09:33 +0200 | [diff] [blame^] | 298 | /** Returns the list of bitrate limits. */ |
Sergey Silkin | be0adee | 2019-06-26 14:11:03 +0200 | [diff] [blame] | 299 | @CalledByNative |
Sergey Silkin | 3d642f8 | 2019-07-03 15:09:33 +0200 | [diff] [blame^] | 300 | default ResolutionBitrateLimits[] getResolutionBitrateLimits() { |
Sergey Silkin | be0adee | 2019-06-26 14:11:03 +0200 | [diff] [blame] | 301 | // TODO(ssilkin): Update downstream projects and remove default implementation. |
Sergey Silkin | 3d642f8 | 2019-07-03 15:09:33 +0200 | [diff] [blame^] | 302 | ResolutionBitrateLimits bitrate_limits[] = {}; |
| 303 | return bitrate_limits; |
Sergey Silkin | be0adee | 2019-06-26 14:11:03 +0200 | [diff] [blame] | 304 | } |
| 305 | |
Sami Kalliomäki | 5f5fc68 | 2017-10-19 11:34:08 +0200 | [diff] [blame] | 306 | /** |
| 307 | * Should return a descriptive name for the implementation. Gets called once and cached. May be |
| 308 | * called from arbitrary thread. |
| 309 | */ |
Magnus Jedvert | 56231d0 | 2017-10-31 17:47:06 +0100 | [diff] [blame] | 310 | @CalledByNative String getImplementationName(); |
Sami Kalliomäki | e2410e9 | 2017-06-02 14:46:12 +0200 | [diff] [blame] | 311 | } |