Add support to adapt video without preserving aspect ratio
This is implemented by allowing users to set two different aspect
ratios, one for landscape input and one for portrait input. This extra
control might be useful in other scenarios as well.
Bug: webrtc:9903
Change-Id: I91676737f4aa1f5d94cfe79ac51d5f866779945b
Reviewed-on: https://webrtc-review.googlesource.com/c/108086
Reviewed-by: Magnus Jedvert <magjed@webrtc.org>
Reviewed-by: Sami Kalliomäki <sakal@webrtc.org>
Commit-Queue: Magnus Jedvert <magjed@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#25387}diff --git a/sdk/android/api/org/webrtc/VideoSource.java b/sdk/android/api/org/webrtc/VideoSource.java
index 7f9f3e5..a8ef662 100644
--- a/sdk/android/api/org/webrtc/VideoSource.java
+++ b/sdk/android/api/org/webrtc/VideoSource.java
@@ -30,7 +30,20 @@
* maintain the input orientation, so it doesn't matter if e.g. 1280x720 or 720x1280 is requested.
*/
public void adaptOutputFormat(int width, int height, int fps) {
- nativeAdaptOutputFormat(getNativeVideoTrackSource(), width, height, fps);
+ final int maxSide = Math.max(width, height);
+ final int minSide = Math.min(width, height);
+ adaptOutputFormat(maxSide, minSide, minSide, maxSide, fps);
+ }
+
+ /**
+ * Same as above, but allows setting two different target resolutions depending on incoming
+ * frame orientation. This gives more fine-grained control and can e.g. be used to force landscape
+ * video to be cropped to portrait video.
+ */
+ public void adaptOutputFormat(
+ int landscapeWidth, int landscapeHeight, int portraitWidth, int portraitHeight, int fps) {
+ nativeAdaptOutputFormat(getNativeVideoTrackSource(), landscapeWidth, landscapeHeight,
+ portraitWidth, portraitHeight, fps);
}
public CapturerObserver getCapturerObserver() {
@@ -44,5 +57,6 @@
// Returns source->internal() from webrtc::VideoTrackSourceProxy.
private static native long nativeGetInternalSource(long source);
- private static native void nativeAdaptOutputFormat(long source, int width, int height, int fps);
+ private static native void nativeAdaptOutputFormat(long source, int landscapeWidth,
+ int landscapeHeight, int portraitWidth, int portraitHeight, int fps);
}
diff --git a/sdk/android/instrumentationtests/src/org/webrtc/Camera1CapturerUsingByteBufferTest.java b/sdk/android/instrumentationtests/src/org/webrtc/Camera1CapturerUsingByteBufferTest.java
index ef76691..17b0977 100644
--- a/sdk/android/instrumentationtests/src/org/webrtc/Camera1CapturerUsingByteBufferTest.java
+++ b/sdk/android/instrumentationtests/src/org/webrtc/Camera1CapturerUsingByteBufferTest.java
@@ -169,6 +169,14 @@
fixtures.scaleCameraOutput();
}
+ // This test that frames forwarded to a renderer is cropped to a new orientation if
+ // adaptOutputFormat is called in such a way. This test both Java and C++ parts of of the stack.
+ @Test
+ @MediumTest
+ public void testCropCameraOutput() throws InterruptedException {
+ fixtures.cropCameraOutput();
+ }
+
// This test that an error is reported if the camera is already opened
// when CameraVideoCapturer is started.
@Test
diff --git a/sdk/android/instrumentationtests/src/org/webrtc/Camera1CapturerUsingTextureTest.java b/sdk/android/instrumentationtests/src/org/webrtc/Camera1CapturerUsingTextureTest.java
index 28fa825..4dc0037 100644
--- a/sdk/android/instrumentationtests/src/org/webrtc/Camera1CapturerUsingTextureTest.java
+++ b/sdk/android/instrumentationtests/src/org/webrtc/Camera1CapturerUsingTextureTest.java
@@ -172,6 +172,14 @@
fixtures.scaleCameraOutput();
}
+ // This test that frames forwarded to a renderer is cropped to a new orientation if
+ // adaptOutputFormat is called in such a way. This test both Java and C++ parts of of the stack.
+ @Test
+ @MediumTest
+ public void testCropCameraOutput() throws InterruptedException {
+ fixtures.cropCameraOutput();
+ }
+
// This test that an error is reported if the camera is already opened
// when CameraVideoCapturer is started.
@Test
diff --git a/sdk/android/instrumentationtests/src/org/webrtc/Camera2CapturerTest.java b/sdk/android/instrumentationtests/src/org/webrtc/Camera2CapturerTest.java
index 77e8d77..fba9431 100644
--- a/sdk/android/instrumentationtests/src/org/webrtc/Camera2CapturerTest.java
+++ b/sdk/android/instrumentationtests/src/org/webrtc/Camera2CapturerTest.java
@@ -302,6 +302,14 @@
fixtures.scaleCameraOutput();
}
+ // This test that frames forwarded to a renderer is cropped to a new orientation if
+ // adaptOutputFormat is called in such a way. This test both Java and C++ parts of of the stack.
+ @Test
+ @MediumTest
+ public void testCropCameraOutput() throws InterruptedException {
+ fixtures.cropCameraOutput();
+ }
+
// This test that an error is reported if the camera is already opened
// when CameraVideoCapturer is started.
@Test
diff --git a/sdk/android/instrumentationtests/src/org/webrtc/CameraVideoCapturerTestFixtures.java b/sdk/android/instrumentationtests/src/org/webrtc/CameraVideoCapturerTestFixtures.java
index dd0d780..b47fee0 100644
--- a/sdk/android/instrumentationtests/src/org/webrtc/CameraVideoCapturerTestFixtures.java
+++ b/sdk/android/instrumentationtests/src/org/webrtc/CameraVideoCapturerTestFixtures.java
@@ -688,6 +688,49 @@
assertTrue(gotExpectedResolution);
}
+ public void cropCameraOutput() throws InterruptedException {
+ final CapturerInstance capturerInstance = createCapturer(false /* initialize */);
+ final VideoTrackWithRenderer videoTrackWithRenderer =
+ createVideoTrackWithRenderer(capturerInstance.capturer);
+ assertTrue(videoTrackWithRenderer.rendererCallbacks.waitForNextFrameToRender() > 0);
+
+ final int startWidth = videoTrackWithRenderer.rendererCallbacks.frameWidth();
+ final int startHeight = videoTrackWithRenderer.rendererCallbacks.frameHeight();
+ final int frameRate = 30;
+ final int cropWidth;
+ final int cropHeight;
+ if (startWidth > startHeight) {
+ // Landscape input, request portrait output.
+ cropWidth = 360;
+ cropHeight = 640;
+ } else {
+ // Portrait input, request landscape output.
+ cropWidth = 640;
+ cropHeight = 630;
+ }
+
+ // Request different output orientation than input.
+ videoTrackWithRenderer.source.adaptOutputFormat(
+ cropWidth, cropHeight, cropWidth, cropHeight, frameRate);
+
+ boolean gotExpectedOrientation = false;
+ int numberOfInspectedFrames = 0;
+
+ do {
+ videoTrackWithRenderer.rendererCallbacks.waitForNextFrameToRender();
+ ++numberOfInspectedFrames;
+
+ gotExpectedOrientation = (cropWidth > cropHeight)
+ == (videoTrackWithRenderer.rendererCallbacks.frameWidth()
+ > videoTrackWithRenderer.rendererCallbacks.frameHeight());
+ } while (!gotExpectedOrientation && numberOfInspectedFrames < 30);
+
+ disposeCapturer(capturerInstance);
+ disposeVideoTrackWithRenderer(videoTrackWithRenderer);
+
+ assertTrue(gotExpectedOrientation);
+ }
+
public void startWhileCameraIsAlreadyOpen() throws InterruptedException {
final String cameraName = testObjectFactory.getNameOfBackFacingDevice();
// At this point camera is not actually opened.
diff --git a/sdk/android/src/jni/androidvideotracksource.cc b/sdk/android/src/jni/androidvideotracksource.cc
index 41d4278..aa459e1 100644
--- a/sdk/android/src/jni/androidvideotracksource.cc
+++ b/sdk/android/src/jni/androidvideotracksource.cc
@@ -84,10 +84,19 @@
int crop_x;
int crop_y;
- if (!AdaptFrame(width, height, camera_time_us, &adapted_width,
- &adapted_height, &crop_width, &crop_height, &crop_x,
- &crop_y)) {
- return;
+ if (rotation % 180 == 0) {
+ if (!AdaptFrame(width, height, camera_time_us, &adapted_width,
+ &adapted_height, &crop_width, &crop_height, &crop_x,
+ &crop_y)) {
+ return;
+ }
+ } else {
+ // Swap all width/height and x/y.
+ if (!AdaptFrame(height, width, camera_time_us, &adapted_height,
+ &adapted_width, &crop_height, &crop_width, &crop_y,
+ &crop_x)) {
+ return;
+ }
}
rtc::scoped_refptr<VideoFrameBuffer> buffer =
@@ -103,12 +112,16 @@
OnFrame(VideoFrame(buffer, rotation, translated_camera_time_us));
}
-void AndroidVideoTrackSource::OnOutputFormatRequest(int width,
- int height,
+void AndroidVideoTrackSource::OnOutputFormatRequest(int landscape_width,
+ int landscape_height,
+ int portrait_width,
+ int portrait_height,
int fps) {
- cricket::VideoFormat format(width, height,
- cricket::VideoFormat::FpsToInterval(fps), 0);
- video_adapter()->OnOutputFormatRequest(format);
+ video_adapter()->OnOutputFormatRequest(
+ std::make_pair(landscape_width, landscape_height),
+ landscape_width * landscape_height,
+ std::make_pair(portrait_width, portrait_height),
+ portrait_width * portrait_height, fps);
}
} // namespace jni
diff --git a/sdk/android/src/jni/androidvideotracksource.h b/sdk/android/src/jni/androidvideotracksource.h
index 8f092c1..4c87432 100644
--- a/sdk/android/src/jni/androidvideotracksource.h
+++ b/sdk/android/src/jni/androidvideotracksource.h
@@ -53,7 +53,11 @@
VideoRotation rotation,
const JavaRef<jobject>& j_video_frame_buffer);
- void OnOutputFormatRequest(int width, int height, int fps);
+ void OnOutputFormatRequest(int landscape_width,
+ int landscape_height,
+ int portrait_width,
+ int portrait_height,
+ int fps);
private:
rtc::Thread* signaling_thread_;
diff --git a/sdk/android/src/jni/videosource.cc b/sdk/android/src/jni/videosource.cc
index 09b538e..37ba0b0 100644
--- a/sdk/android/src/jni/videosource.cc
+++ b/sdk/android/src/jni/videosource.cc
@@ -33,13 +33,16 @@
static void JNI_VideoSource_AdaptOutputFormat(JNIEnv* jni,
const JavaParamRef<jclass>&,
jlong j_source,
- jint j_width,
- jint j_height,
+ jint j_landscape_width,
+ jint j_landscape_height,
+ jint j_portrait_width,
+ jint j_portrait_height,
jint j_fps) {
RTC_LOG(LS_INFO) << "VideoSource_nativeAdaptOutputFormat";
AndroidVideoTrackSource* source =
AndroidVideoTrackSourceFromJavaProxy(j_source);
- source->OnOutputFormatRequest(j_width, j_height, j_fps);
+ source->OnOutputFormatRequest(j_landscape_width, j_landscape_height,
+ j_portrait_width, j_portrait_height, j_fps);
}
} // namespace jni