Android: Add video processing interface
This CL adds an API for injecting video processing after the WebRTC
CPU and QP scaling step.
Bug: webrtc:10247
Change-Id: I776498e1e9113f50e953ee411bbb31f181863312
Reviewed-on: https://webrtc-review.googlesource.com/c/119953
Commit-Queue: Magnus Jedvert <magjed@webrtc.org>
Reviewed-by: Sami Kalliomäki <sakal@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#26740}diff --git a/sdk/android/api/org/webrtc/VideoSource.java b/sdk/android/api/org/webrtc/VideoSource.java
index 0939457..847a7bd 100644
--- a/sdk/android/api/org/webrtc/VideoSource.java
+++ b/sdk/android/api/org/webrtc/VideoSource.java
@@ -30,15 +30,31 @@
}
private final NativeAndroidVideoTrackSource nativeAndroidVideoTrackSource;
+ private final Object videoProcessorLock = new Object();
+ @Nullable private VideoProcessor videoProcessor;
+ private boolean isCapturerRunning;
+
private final CapturerObserver capturerObserver = new CapturerObserver() {
@Override
public void onCapturerStarted(boolean success) {
nativeAndroidVideoTrackSource.setState(success);
+ synchronized (videoProcessorLock) {
+ isCapturerRunning = success;
+ if (videoProcessor != null) {
+ videoProcessor.onCapturerStarted(success);
+ }
+ }
}
@Override
public void onCapturerStopped() {
nativeAndroidVideoTrackSource.setState(/* isLive= */ false);
+ synchronized (videoProcessorLock) {
+ isCapturerRunning = false;
+ if (videoProcessor != null) {
+ videoProcessor.onCapturerStopped();
+ }
+ }
}
@Override
@@ -53,9 +69,17 @@
final VideoFrame.Buffer adaptedBuffer =
frame.getBuffer().cropAndScale(parameters.cropX, parameters.cropY, parameters.cropWidth,
parameters.cropHeight, parameters.scaleWidth, parameters.scaleHeight);
- // TODO(magjed): Add video processing hook here.
- nativeAndroidVideoTrackSource.onFrameCaptured(
- new VideoFrame(adaptedBuffer, frame.getRotation(), parameters.timestampNs));
+ final VideoFrame adaptedFrame =
+ new VideoFrame(adaptedBuffer, frame.getRotation(), parameters.timestampNs);
+
+ synchronized (videoProcessorLock) {
+ if (videoProcessor != null) {
+ videoProcessor.onFrameCaptured(adaptedFrame);
+ adaptedBuffer.release();
+ return;
+ }
+ }
+ nativeAndroidVideoTrackSource.onFrameCaptured(adaptedFrame);
adaptedBuffer.release();
}
};
@@ -98,6 +122,31 @@
maxLandscapePixelCount, targetPortraitAspectRatio, maxPortraitPixelCount, maxFps);
}
+ /**
+ * Hook for injecting a custom video processor before frames are passed onto WebRTC. The frames
+ * will be cropped and scaled depending on CPU and network conditions before they are passed to
+ * the video processor. Frames will be delivered to the video processor on the same thread they
+ * are passed to this object. The video processor is allowed to deliver the processed frames
+ * back on any thread.
+ */
+ public void setVideoProcessor(@Nullable VideoProcessor newVideoProcessor) {
+ synchronized (videoProcessorLock) {
+ if (videoProcessor != null) {
+ videoProcessor.setSink(/* sink= */ null);
+ if (isCapturerRunning) {
+ videoProcessor.onCapturerStopped();
+ }
+ }
+ videoProcessor = newVideoProcessor;
+ if (newVideoProcessor != null) {
+ newVideoProcessor.setSink(nativeAndroidVideoTrackSource::onFrameCaptured);
+ if (isCapturerRunning) {
+ newVideoProcessor.onCapturerStarted(/* success= */ true);
+ }
+ }
+ }
+ }
+
public CapturerObserver getCapturerObserver() {
return capturerObserver;
}
@@ -106,4 +155,10 @@
long getNativeVideoTrackSource() {
return getNativeMediaSource();
}
+
+ @Override
+ public void dispose() {
+ setVideoProcessor(/* newVideoProcessor= */ null);
+ super.dispose();
+ }
}