Android: Add error callback for GL_OUT_OF_MEMORY in EglRenderer

Encountering GL_OUT_OF_MEMORY is relatively common and we should give
clients a chance to deal with it in a non-fatal way.

Bug: webrtc:8154
Change-Id: Ifa9ca74392f21083692b02a5144dc5632a88d34d
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/144561
Commit-Queue: Magnus Jedvert <magjed@webrtc.org>
Reviewed-by: Sami Kalliomäki <sakal@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#28495}
diff --git a/sdk/android/api/org/webrtc/EglRenderer.java b/sdk/android/api/org/webrtc/EglRenderer.java
index 2ab2779..950f0b5 100644
--- a/sdk/android/api/org/webrtc/EglRenderer.java
+++ b/sdk/android/api/org/webrtc/EglRenderer.java
@@ -37,6 +37,12 @@
 
   public interface FrameListener { void onFrame(Bitmap frame); }
 
+  /** Callback for clients to be notified about errors encountered during rendering. */
+  public static interface ErrorCallback {
+    /** Called if GLES20.GL_OUT_OF_MEMORY is encountered during rendering. */
+    void onGlOutOfMemory();
+  }
+
   private static class FrameListenerAndParams {
     public final FrameListener listener;
     public final float scale;
@@ -112,6 +118,8 @@
 
   private final ArrayList<FrameListenerAndParams> frameListeners = new ArrayList<>();
 
+  private volatile ErrorCallback errorCallback;
+
   // Variables for fps reduction.
   private final Object fpsReductionLock = new Object();
   // Time for when next frame should be rendered.
@@ -485,6 +493,11 @@
     ThreadUtils.awaitUninterruptibly(latch);
   }
 
+  /** Can be set in order to be notified about errors encountered during rendering. */
+  public void setErrorCallback(ErrorCallback errorCallback) {
+    this.errorCallback = errorCallback;
+  }
+
   // VideoSink interface.
   @Override
   public void onFrame(VideoFrame frame) {
@@ -642,29 +655,44 @@
     drawMatrix.preScale(scaleX, scaleY);
     drawMatrix.preTranslate(-0.5f, -0.5f);
 
-    if (shouldRenderFrame) {
-      GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */);
-      GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
-      frameDrawer.drawFrame(frame, drawer, drawMatrix, 0 /* viewportX */, 0 /* viewportY */,
-          eglBase.surfaceWidth(), eglBase.surfaceHeight());
+    try {
+      if (shouldRenderFrame) {
+        GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */);
+        GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
+        frameDrawer.drawFrame(frame, drawer, drawMatrix, 0 /* viewportX */, 0 /* viewportY */,
+            eglBase.surfaceWidth(), eglBase.surfaceHeight());
 
-      final long swapBuffersStartTimeNs = System.nanoTime();
-      if (usePresentationTimeStamp) {
-        eglBase.swapBuffers(frame.getTimestampNs());
-      } else {
-        eglBase.swapBuffers();
+        final long swapBuffersStartTimeNs = System.nanoTime();
+        if (usePresentationTimeStamp) {
+          eglBase.swapBuffers(frame.getTimestampNs());
+        } else {
+          eglBase.swapBuffers();
+        }
+
+        final long currentTimeNs = System.nanoTime();
+        synchronized (statisticsLock) {
+          ++framesRendered;
+          renderTimeNs += (currentTimeNs - startTimeNs);
+          renderSwapBufferTimeNs += (currentTimeNs - swapBuffersStartTimeNs);
+        }
       }
 
-      final long currentTimeNs = System.nanoTime();
-      synchronized (statisticsLock) {
-        ++framesRendered;
-        renderTimeNs += (currentTimeNs - startTimeNs);
-        renderSwapBufferTimeNs += (currentTimeNs - swapBuffersStartTimeNs);
+      notifyCallbacks(frame, shouldRenderFrame);
+    } catch (GlUtil.GlOutOfMemoryException e) {
+      logE("Error while drawing frame", e);
+      final ErrorCallback errorCallback = this.errorCallback;
+      if (errorCallback != null) {
+        errorCallback.onGlOutOfMemory();
       }
+      // Attempt to free up some resources.
+      drawer.release();
+      frameDrawer.release();
+      bitmapTextureFramebuffer.release();
+      // Continue here on purpose and retry again for next frame. In worst case, this is a continous
+      // problem and no more frames will be drawn.
+    } finally {
+      frame.release();
     }
-
-    notifyCallbacks(frame, shouldRenderFrame);
-    frame.release();
   }
 
   private void notifyCallbacks(VideoFrame frame, boolean wasRendered) {
@@ -743,6 +771,10 @@
     }
   }
 
+  private void logE(String string, Throwable e) {
+    Logging.e(TAG, name + string, e);
+  }
+
   private void logD(String string) {
     Logging.d(TAG, name + string);
   }
diff --git a/sdk/android/api/org/webrtc/GlUtil.java b/sdk/android/api/org/webrtc/GlUtil.java
index 6f5e605..bdafe81 100644
--- a/sdk/android/api/org/webrtc/GlUtil.java
+++ b/sdk/android/api/org/webrtc/GlUtil.java
@@ -22,11 +22,19 @@
 public class GlUtil {
   private GlUtil() {}
 
+  public static class GlOutOfMemoryException extends RuntimeException {
+    public GlOutOfMemoryException(String msg) {
+      super(msg);
+    }
+  }
+
   // Assert that no OpenGL ES 2.0 error has been raised.
   public static void checkNoGLES2Error(String msg) {
     int error = GLES20.glGetError();
     if (error != GLES20.GL_NO_ERROR) {
-      throw new RuntimeException(msg + ": GLES20 error: " + error);
+      throw error == GLES20.GL_OUT_OF_MEMORY
+          ? new GlOutOfMemoryException(msg)
+          : new RuntimeException(msg + ": GLES20 error: " + error);
     }
   }