Interface for monitoring ref counts of texture buffers created by SurfaceTextureHelper.
Bug: b/139745386
Change-Id: I095d6b2862dac55044af5852098fb1c38e8738cf
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/150649
Commit-Queue: Sami Kalliomäki <sakal@webrtc.org>
Reviewed-by: Alex Glaznev <glaznev@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#29024}
diff --git a/sdk/android/api/org/webrtc/SurfaceTextureHelper.java b/sdk/android/api/org/webrtc/SurfaceTextureHelper.java
index b8f5624..3522a87 100644
--- a/sdk/android/api/org/webrtc/SurfaceTextureHelper.java
+++ b/sdk/android/api/org/webrtc/SurfaceTextureHelper.java
@@ -11,7 +11,6 @@
package org.webrtc;
import android.annotation.TargetApi;
-import android.graphics.Matrix;
import android.graphics.SurfaceTexture;
import android.opengl.GLES11Ext;
import android.opengl.GLES20;
@@ -19,9 +18,9 @@
import android.os.Handler;
import android.os.HandlerThread;
import android.support.annotation.Nullable;
-import java.nio.ByteBuffer;
import java.util.concurrent.Callable;
-import org.webrtc.EglBase;
+import org.webrtc.EglBase.Context;
+import org.webrtc.TextureBufferImpl.RefCountMonitor;
import org.webrtc.VideoFrame.TextureBuffer;
/**
@@ -32,6 +31,21 @@
* resources once the texture frame is released.
*/
public class SurfaceTextureHelper {
+ /**
+ * Interface for monitoring texture buffers created from this SurfaceTexture. Since only one
+ * texture buffer can exist at a time, this can be used to monitor for stuck frames.
+ */
+ public interface FrameRefMonitor {
+ /** A new frame was created. New frames start with ref count of 1. */
+ void onNewBuffer(TextureBuffer textureBuffer);
+ /** Ref count of the frame was incremented by the calling thread. */
+ void onRetainBuffer(TextureBuffer textureBuffer);
+ /** Ref count of the frame was decremented by the calling thread. */
+ void onReleaseBuffer(TextureBuffer textureBuffer);
+ /** Frame was destroyed (ref count reached 0). */
+ void onDestroyBuffer(TextureBuffer textureBuffer);
+ }
+
private static final String TAG = "SurfaceTextureHelper";
/**
* Construct a new SurfaceTextureHelper sharing OpenGL resources with |sharedContext|. A dedicated
@@ -43,8 +57,8 @@
* closer to actual creation time.
*/
public static SurfaceTextureHelper create(final String threadName,
- final EglBase.Context sharedContext, boolean alignTimestamps,
- final YuvConverter yuvConverter) {
+ final EglBase.Context sharedContext, boolean alignTimestamps, final YuvConverter yuvConverter,
+ FrameRefMonitor frameRefMonitor) {
final HandlerThread thread = new HandlerThread(threadName);
thread.start();
final Handler handler = new Handler(thread.getLooper());
@@ -58,7 +72,8 @@
@Override
public SurfaceTextureHelper call() {
try {
- return new SurfaceTextureHelper(sharedContext, handler, alignTimestamps, yuvConverter);
+ return new SurfaceTextureHelper(
+ sharedContext, handler, alignTimestamps, yuvConverter, frameRefMonitor);
} catch (RuntimeException e) {
Logging.e(TAG, threadName + " create failure", e);
return null;
@@ -70,29 +85,67 @@
/**
* Same as above with alignTimestamps set to false and yuvConverter set to new YuvConverter.
*
- * @see #create(String, EglBase.Context, boolean, YuvConverter)
+ * @see #create(String, EglBase.Context, boolean, YuvConverter, FrameRefMonitor)
*/
public static SurfaceTextureHelper create(
final String threadName, final EglBase.Context sharedContext) {
- return create(threadName, sharedContext, /* alignTimestamps= */ false, new YuvConverter());
+ return create(threadName, sharedContext, /* alignTimestamps= */ false, new YuvConverter(),
+ /*frameRefMonitor=*/null);
}
/**
* Same as above with yuvConverter set to new YuvConverter.
*
- * @see #create(String, EglBase.Context, boolean, YuvConverter)
+ * @see #create(String, EglBase.Context, boolean, YuvConverter, FrameRefMonitor)
*/
public static SurfaceTextureHelper create(
final String threadName, final EglBase.Context sharedContext, boolean alignTimestamps) {
- return create(threadName, sharedContext, alignTimestamps, new YuvConverter());
+ return create(
+ threadName, sharedContext, alignTimestamps, new YuvConverter(), /*frameRefMonitor=*/null);
}
+ /**
+ * Create a SurfaceTextureHelper without frame ref monitor.
+ *
+ * @see #create(String, EglBase.Context, boolean, YuvConverter, FrameRefMonitor)
+ */
+ public static SurfaceTextureHelper create(final String threadName,
+ final EglBase.Context sharedContext, boolean alignTimestamps, YuvConverter yuvConverter) {
+ return create(
+ threadName, sharedContext, alignTimestamps, yuvConverter, /*frameRefMonitor=*/null);
+ }
+
+ private final RefCountMonitor textureRefCountMonitor = new RefCountMonitor() {
+ @Override
+ public void onRetain(TextureBufferImpl textureBuffer) {
+ if (frameRefMonitor != null) {
+ frameRefMonitor.onRetainBuffer(textureBuffer);
+ }
+ }
+
+ @Override
+ public void onRelease(TextureBufferImpl textureBuffer) {
+ if (frameRefMonitor != null) {
+ frameRefMonitor.onReleaseBuffer(textureBuffer);
+ }
+ }
+
+ @Override
+ public void onDestroy(TextureBufferImpl textureBuffer) {
+ returnTextureFrame();
+ if (frameRefMonitor != null) {
+ frameRefMonitor.onDestroyBuffer(textureBuffer);
+ }
+ }
+ };
+
private final Handler handler;
private final EglBase eglBase;
private final SurfaceTexture surfaceTexture;
private final int oesTextureId;
private final YuvConverter yuvConverter;
@Nullable private final TimestampAligner timestampAligner;
+ private final FrameRefMonitor frameRefMonitor;
// These variables are only accessed from the |handler| thread.
@Nullable private VideoSink listener;
@@ -121,14 +174,15 @@
}
};
- private SurfaceTextureHelper(EglBase.Context sharedContext, Handler handler,
- boolean alignTimestamps, YuvConverter yuvConverter) {
+ private SurfaceTextureHelper(Context sharedContext, Handler handler, boolean alignTimestamps,
+ YuvConverter yuvConverter, FrameRefMonitor frameRefMonitor) {
if (handler.getLooper().getThread() != Thread.currentThread()) {
throw new IllegalStateException("SurfaceTextureHelper must be created on the handler thread");
}
this.handler = handler;
this.timestampAligner = alignTimestamps ? new TimestampAligner() : null;
this.yuvConverter = yuvConverter;
+ this.frameRefMonitor = frameRefMonitor;
eglBase = EglBase.create(sharedContext, EglBase.CONFIG_PIXEL_BUFFER);
try {
@@ -304,12 +358,15 @@
if (timestampAligner != null) {
timestampNs = timestampAligner.translateTimestamp(timestampNs);
}
- final VideoFrame.Buffer buffer =
+ final VideoFrame.TextureBuffer buffer =
new TextureBufferImpl(textureWidth, textureHeight, TextureBuffer.Type.OES, oesTextureId,
RendererCommon.convertMatrixToAndroidGraphicsMatrix(transformMatrix), handler,
- yuvConverter, this ::returnTextureFrame);
+ yuvConverter, textureRefCountMonitor);
+ if (frameRefMonitor != null) {
+ frameRefMonitor.onNewBuffer(buffer);
+ }
final VideoFrame frame = new VideoFrame(buffer, frameRotation, timestampNs);
- ((VideoSink) listener).onFrame(frame);
+ listener.onFrame(frame);
frame.release();
}
diff --git a/sdk/android/api/org/webrtc/TextureBufferImpl.java b/sdk/android/api/org/webrtc/TextureBufferImpl.java
index a24f284..3d3bbab 100644
--- a/sdk/android/api/org/webrtc/TextureBufferImpl.java
+++ b/sdk/android/api/org/webrtc/TextureBufferImpl.java
@@ -19,6 +19,12 @@
* release callback. ToI420() is implemented by providing a Handler and a YuvConverter.
*/
public class TextureBufferImpl implements VideoFrame.TextureBuffer {
+ interface RefCountMonitor {
+ void onRetain(TextureBufferImpl textureBuffer);
+ void onRelease(TextureBufferImpl textureBuffer);
+ void onDestroy(TextureBufferImpl textureBuffer);
+ }
+
// This is the full resolution the texture has in memory after applying the transformation matrix
// that might include cropping. This resolution is useful to know when sampling the texture to
// avoid downscaling artifacts.
@@ -33,24 +39,34 @@
private final Handler toI420Handler;
private final YuvConverter yuvConverter;
private final RefCountDelegate refCountDelegate;
+ private final @Nullable RefCountMonitor refCountMonitor;
public TextureBufferImpl(int width, int height, Type type, int id, Matrix transformMatrix,
Handler toI420Handler, YuvConverter yuvConverter, @Nullable Runnable releaseCallback) {
- this.unscaledWidth = width;
- this.unscaledHeight = height;
- this.width = width;
- this.height = height;
- this.type = type;
- this.id = id;
- this.transformMatrix = transformMatrix;
- this.toI420Handler = toI420Handler;
- this.yuvConverter = yuvConverter;
- this.refCountDelegate = new RefCountDelegate(releaseCallback);
+ this(width, height, width, height, type, id, transformMatrix, toI420Handler, yuvConverter,
+ new RefCountMonitor() {
+ @Override
+ public void onRetain(TextureBufferImpl textureBuffer) {}
+
+ @Override
+ public void onRelease(TextureBufferImpl textureBuffer) {}
+
+ @Override
+ public void onDestroy(TextureBufferImpl textureBuffer) {
+ releaseCallback.run();
+ }
+ });
+ }
+
+ TextureBufferImpl(int width, int height, Type type, int id, Matrix transformMatrix,
+ Handler toI420Handler, YuvConverter yuvConverter, RefCountMonitor refCountMonitor) {
+ this(width, height, width, height, type, id, transformMatrix, toI420Handler, yuvConverter,
+ refCountMonitor);
}
private TextureBufferImpl(int unscaledWidth, int unscaledHeight, int width, int height, Type type,
int id, Matrix transformMatrix, Handler toI420Handler, YuvConverter yuvConverter,
- @Nullable Runnable releaseCallback) {
+ RefCountMonitor refCountMonitor) {
this.unscaledWidth = unscaledWidth;
this.unscaledHeight = unscaledHeight;
this.width = width;
@@ -60,7 +76,8 @@
this.transformMatrix = transformMatrix;
this.toI420Handler = toI420Handler;
this.yuvConverter = yuvConverter;
- this.refCountDelegate = new RefCountDelegate(releaseCallback);
+ this.refCountDelegate = new RefCountDelegate(() -> refCountMonitor.onDestroy(this));
+ this.refCountMonitor = refCountMonitor;
}
@Override
@@ -96,11 +113,13 @@
@Override
public void retain() {
+ refCountMonitor.onRetain(this);
refCountDelegate.retain();
}
@Override
public void release() {
+ refCountMonitor.onRelease(this);
refCountDelegate.release();
}
@@ -161,6 +180,21 @@
newMatrix.preConcat(transformMatrix);
retain();
return new TextureBufferImpl(unscaledWidth, unscaledHeight, scaledWidth, scaledHeight, type, id,
- newMatrix, toI420Handler, yuvConverter, this ::release);
+ newMatrix, toI420Handler, yuvConverter, new RefCountMonitor() {
+ @Override
+ public void onRetain(TextureBufferImpl textureBuffer) {
+ refCountMonitor.onRetain(TextureBufferImpl.this);
+ }
+
+ @Override
+ public void onRelease(TextureBufferImpl textureBuffer) {
+ refCountMonitor.onRelease(TextureBufferImpl.this);
+ }
+
+ @Override
+ public void onDestroy(TextureBufferImpl textureBuffer) {
+ release();
+ }
+ });
}
}