blob: c6e98123aa048a6d411ea24cc6f36556f495be98 [file] [log] [blame]
magjeddf494b02016-10-07 05:32:35 -07001/*
2 * Copyright 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11package org.webrtc;
12
sakalfb0c5732016-11-03 09:15:34 -070013import android.graphics.Bitmap;
sakal6bdcefc2017-08-15 01:56:02 -070014import android.graphics.Matrix;
magjed9ab8a182016-10-20 03:18:09 -070015import android.graphics.SurfaceTexture;
magjeddf494b02016-10-07 05:32:35 -070016import android.opengl.GLES20;
17import android.os.Handler;
18import android.os.HandlerThread;
19import android.os.Looper;
20import android.view.Surface;
sakalfb0c5732016-11-03 09:15:34 -070021import java.nio.ByteBuffer;
22import java.util.ArrayList;
23import java.util.Iterator;
sakal037b93a2017-01-16 04:57:32 -080024import java.util.Locale;
magjeddf494b02016-10-07 05:32:35 -070025import java.util.concurrent.CountDownLatch;
26import java.util.concurrent.TimeUnit;
Sami Kalliomäkie7592d82018-03-22 13:32:44 +010027import javax.annotation.Nullable;
magjeddf494b02016-10-07 05:32:35 -070028
29/**
30 * Implements org.webrtc.VideoRenderer.Callbacks by displaying the video stream on an EGL Surface.
31 * This class is intended to be used as a helper class for rendering on SurfaceViews and
32 * TextureViews.
33 */
sakal6bdcefc2017-08-15 01:56:02 -070034public class EglRenderer implements VideoRenderer.Callbacks, VideoSink {
magjeddf494b02016-10-07 05:32:35 -070035 private static final String TAG = "EglRenderer";
magjed9ab8a182016-10-20 03:18:09 -070036 private static final long LOG_INTERVAL_SEC = 4;
magjeddf494b02016-10-07 05:32:35 -070037
sakalfb0c5732016-11-03 09:15:34 -070038 public interface FrameListener { void onFrame(Bitmap frame); }
39
sakal3a9bc172016-11-30 08:30:05 -080040 private static class FrameListenerAndParams {
sakalfb0c5732016-11-03 09:15:34 -070041 public final FrameListener listener;
sakal3a9bc172016-11-30 08:30:05 -080042 public final float scale;
43 public final RendererCommon.GlDrawer drawer;
sakal8fdf9572017-05-31 02:43:10 -070044 public final boolean applyFpsReduction;
sakalfb0c5732016-11-03 09:15:34 -070045
sakal8fdf9572017-05-31 02:43:10 -070046 public FrameListenerAndParams(FrameListener listener, float scale,
47 RendererCommon.GlDrawer drawer, boolean applyFpsReduction) {
sakalfb0c5732016-11-03 09:15:34 -070048 this.listener = listener;
sakal3a9bc172016-11-30 08:30:05 -080049 this.scale = scale;
50 this.drawer = drawer;
sakal8fdf9572017-05-31 02:43:10 -070051 this.applyFpsReduction = applyFpsReduction;
sakalfb0c5732016-11-03 09:15:34 -070052 }
53 }
54
magjeddf494b02016-10-07 05:32:35 -070055 private class EglSurfaceCreation implements Runnable {
magjed9ab8a182016-10-20 03:18:09 -070056 private Object surface;
magjeddf494b02016-10-07 05:32:35 -070057
Mirko Bonadei12251b62017-11-05 19:35:31 -080058 // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression.
59 @SuppressWarnings("NoSynchronizedMethodCheck")
magjed9ab8a182016-10-20 03:18:09 -070060 public synchronized void setSurface(Object surface) {
magjeddf494b02016-10-07 05:32:35 -070061 this.surface = surface;
62 }
63
64 @Override
Mirko Bonadei12251b62017-11-05 19:35:31 -080065 // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression.
66 @SuppressWarnings("NoSynchronizedMethodCheck")
magjeddf494b02016-10-07 05:32:35 -070067 public synchronized void run() {
68 if (surface != null && eglBase != null && !eglBase.hasSurface()) {
magjed9ab8a182016-10-20 03:18:09 -070069 if (surface instanceof Surface) {
70 eglBase.createSurface((Surface) surface);
71 } else if (surface instanceof SurfaceTexture) {
72 eglBase.createSurface((SurfaceTexture) surface);
73 } else {
74 throw new IllegalStateException("Invalid surface: " + surface);
75 }
magjeddf494b02016-10-07 05:32:35 -070076 eglBase.makeCurrent();
77 // Necessary for YUV frames with odd width.
78 GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1);
79 }
80 }
81 }
82
Xiaolei Yu149533a2017-11-03 07:55:01 +080083 protected final String name;
magjeddf494b02016-10-07 05:32:35 -070084
85 // |renderThreadHandler| is a handler for communicating with |renderThread|, and is synchronized
86 // on |handlerLock|.
87 private final Object handlerLock = new Object();
Sami Kalliomäkie7592d82018-03-22 13:32:44 +010088 @Nullable private Handler renderThreadHandler;
magjeddf494b02016-10-07 05:32:35 -070089
sakal3a9bc172016-11-30 08:30:05 -080090 private final ArrayList<FrameListenerAndParams> frameListeners = new ArrayList<>();
sakalfb0c5732016-11-03 09:15:34 -070091
magjed9ab8a182016-10-20 03:18:09 -070092 // Variables for fps reduction.
93 private final Object fpsReductionLock = new Object();
94 // Time for when next frame should be rendered.
95 private long nextFrameTimeNs;
96 // Minimum duration between frames when fps reduction is active, or -1 if video is completely
97 // paused.
98 private long minRenderPeriodNs;
99
magjeddf494b02016-10-07 05:32:35 -0700100 // EGL and GL resources for drawing YUV/OES textures. After initilization, these are only accessed
101 // from the render thread.
Sami Kalliomäkie7592d82018-03-22 13:32:44 +0100102 @Nullable private EglBase eglBase;
magjed7cede372017-09-11 06:12:07 -0700103 private final VideoFrameDrawer frameDrawer = new VideoFrameDrawer();
Sami Kalliomäkie7592d82018-03-22 13:32:44 +0100104 @Nullable private RendererCommon.GlDrawer drawer;
magjed7cede372017-09-11 06:12:07 -0700105 private final Matrix drawMatrix = new Matrix();
magjeddf494b02016-10-07 05:32:35 -0700106
107 // Pending frame to render. Serves as a queue with size 1. Synchronized on |frameLock|.
108 private final Object frameLock = new Object();
Sami Kalliomäkie7592d82018-03-22 13:32:44 +0100109 @Nullable private VideoFrame pendingFrame;
magjeddf494b02016-10-07 05:32:35 -0700110
111 // These variables are synchronized on |layoutLock|.
112 private final Object layoutLock = new Object();
magjeddf494b02016-10-07 05:32:35 -0700113 private float layoutAspectRatio;
114 // If true, mirrors the video stream horizontally.
115 private boolean mirror;
116
117 // These variables are synchronized on |statisticsLock|.
118 private final Object statisticsLock = new Object();
119 // Total number of video frames received in renderFrame() call.
120 private int framesReceived;
121 // Number of video frames dropped by renderFrame() because previous frame has not been rendered
122 // yet.
123 private int framesDropped;
124 // Number of rendered video frames.
125 private int framesRendered;
magjed9ab8a182016-10-20 03:18:09 -0700126 // Start time for counting these statistics, or 0 if we haven't started measuring yet.
127 private long statisticsStartTimeNs;
magjeddf494b02016-10-07 05:32:35 -0700128 // Time in ns spent in renderFrameOnRenderThread() function.
129 private long renderTimeNs;
magjed9ab8a182016-10-20 03:18:09 -0700130 // Time in ns spent by the render thread in the swapBuffers() function.
131 private long renderSwapBufferTimeNs;
magjeddf494b02016-10-07 05:32:35 -0700132
sakalfb0c5732016-11-03 09:15:34 -0700133 // Used for bitmap capturing.
Sami Kalliomäkie7592d82018-03-22 13:32:44 +0100134 @Nullable private GlTextureFrameBuffer bitmapTextureFramebuffer;
sakalfb0c5732016-11-03 09:15:34 -0700135
magjed9ab8a182016-10-20 03:18:09 -0700136 private final Runnable logStatisticsRunnable = new Runnable() {
137 @Override
138 public void run() {
139 logStatistics();
140 synchronized (handlerLock) {
141 if (renderThreadHandler != null) {
142 renderThreadHandler.removeCallbacks(logStatisticsRunnable);
143 renderThreadHandler.postDelayed(
144 logStatisticsRunnable, TimeUnit.SECONDS.toMillis(LOG_INTERVAL_SEC));
145 }
146 }
147 }
148 };
149
magjeddf494b02016-10-07 05:32:35 -0700150 private final EglSurfaceCreation eglSurfaceCreationRunnable = new EglSurfaceCreation();
151
152 /**
153 * Standard constructor. The name will be used for the render thread name and included when
154 * logging. In order to render something, you must first call init() and createEglSurface.
155 */
156 public EglRenderer(String name) {
157 this.name = name;
158 }
159
160 /**
161 * Initialize this class, sharing resources with |sharedContext|. The custom |drawer| will be used
162 * for drawing frames on the EGLSurface. This class is responsible for calling release() on
163 * |drawer|. It is allowed to call init() to reinitialize the renderer after a previous
164 * init()/release() cycle.
165 */
Sami Kalliomäkie7592d82018-03-22 13:32:44 +0100166 public void init(@Nullable final EglBase.Context sharedContext, final int[] configAttributes,
magjeddf494b02016-10-07 05:32:35 -0700167 RendererCommon.GlDrawer drawer) {
168 synchronized (handlerLock) {
169 if (renderThreadHandler != null) {
170 throw new IllegalStateException(name + "Already initialized");
171 }
172 logD("Initializing EglRenderer");
173 this.drawer = drawer;
174
175 final HandlerThread renderThread = new HandlerThread(name + "EglRenderer");
176 renderThread.start();
177 renderThreadHandler = new Handler(renderThread.getLooper());
178 // Create EGL context on the newly created render thread. It should be possibly to create the
179 // context on this thread and make it current on the render thread, but this causes failure on
180 // some Marvel based JB devices. https://bugs.chromium.org/p/webrtc/issues/detail?id=6350.
sakalbf080602017-08-11 01:42:43 -0700181 ThreadUtils.invokeAtFrontUninterruptibly(renderThreadHandler, () -> {
182 // If sharedContext is null, then texture frames are disabled. This is typically for old
183 // devices that might not be fully spec compliant, so force EGL 1.0 since EGL 1.4 has
184 // caused trouble on some weird devices.
185 if (sharedContext == null) {
186 logD("EglBase10.create context");
187 eglBase = EglBase.createEgl10(configAttributes);
188 } else {
189 logD("EglBase.create shared context");
190 eglBase = EglBase.create(sharedContext, configAttributes);
magjeddf494b02016-10-07 05:32:35 -0700191 }
192 });
magjed9ab8a182016-10-20 03:18:09 -0700193 renderThreadHandler.post(eglSurfaceCreationRunnable);
194 final long currentTimeNs = System.nanoTime();
195 resetStatistics(currentTimeNs);
196 renderThreadHandler.postDelayed(
197 logStatisticsRunnable, TimeUnit.SECONDS.toMillis(LOG_INTERVAL_SEC));
magjeddf494b02016-10-07 05:32:35 -0700198 }
199 }
200
201 public void createEglSurface(Surface surface) {
magjed9ab8a182016-10-20 03:18:09 -0700202 createEglSurfaceInternal(surface);
203 }
204
205 public void createEglSurface(SurfaceTexture surfaceTexture) {
206 createEglSurfaceInternal(surfaceTexture);
207 }
208
209 private void createEglSurfaceInternal(Object surface) {
magjeddf494b02016-10-07 05:32:35 -0700210 eglSurfaceCreationRunnable.setSurface(surface);
magjed9ab8a182016-10-20 03:18:09 -0700211 postToRenderThread(eglSurfaceCreationRunnable);
magjeddf494b02016-10-07 05:32:35 -0700212 }
213
214 /**
215 * Block until any pending frame is returned and all GL resources released, even if an interrupt
216 * occurs. If an interrupt occurs during release(), the interrupt flag will be set. This function
217 * should be called before the Activity is destroyed and the EGLContext is still valid. If you
218 * don't call this function, the GL resources might leak.
219 */
220 public void release() {
magjed9ab8a182016-10-20 03:18:09 -0700221 logD("Releasing.");
magjeddf494b02016-10-07 05:32:35 -0700222 final CountDownLatch eglCleanupBarrier = new CountDownLatch(1);
223 synchronized (handlerLock) {
224 if (renderThreadHandler == null) {
225 logD("Already released");
226 return;
227 }
magjed9ab8a182016-10-20 03:18:09 -0700228 renderThreadHandler.removeCallbacks(logStatisticsRunnable);
magjeddf494b02016-10-07 05:32:35 -0700229 // Release EGL and GL resources on render thread.
sakalbf080602017-08-11 01:42:43 -0700230 renderThreadHandler.postAtFrontOfQueue(() -> {
231 if (drawer != null) {
232 drawer.release();
233 drawer = null;
magjeddf494b02016-10-07 05:32:35 -0700234 }
magjed7cede372017-09-11 06:12:07 -0700235 frameDrawer.release();
sakalbf080602017-08-11 01:42:43 -0700236 if (bitmapTextureFramebuffer != null) {
237 bitmapTextureFramebuffer.release();
238 bitmapTextureFramebuffer = null;
239 }
240 if (eglBase != null) {
241 logD("eglBase detach and release.");
242 eglBase.detachCurrent();
243 eglBase.release();
244 eglBase = null;
245 }
Sami Kalliomäki8ebac242017-11-08 17:13:13 +0100246 frameListeners.clear();
sakalbf080602017-08-11 01:42:43 -0700247 eglCleanupBarrier.countDown();
magjeddf494b02016-10-07 05:32:35 -0700248 });
249 final Looper renderLooper = renderThreadHandler.getLooper();
250 // TODO(magjed): Replace this post() with renderLooper.quitSafely() when API support >= 18.
sakalbf080602017-08-11 01:42:43 -0700251 renderThreadHandler.post(() -> {
252 logD("Quitting render thread.");
253 renderLooper.quit();
magjeddf494b02016-10-07 05:32:35 -0700254 });
255 // Don't accept any more frames or messages to the render thread.
256 renderThreadHandler = null;
257 }
258 // Make sure the EGL/GL cleanup posted above is executed.
259 ThreadUtils.awaitUninterruptibly(eglCleanupBarrier);
260 synchronized (frameLock) {
261 if (pendingFrame != null) {
sakal6bdcefc2017-08-15 01:56:02 -0700262 pendingFrame.release();
magjeddf494b02016-10-07 05:32:35 -0700263 pendingFrame = null;
264 }
265 }
magjeddf494b02016-10-07 05:32:35 -0700266 logD("Releasing done.");
267 }
268
269 /**
magjed9ab8a182016-10-20 03:18:09 -0700270 * Reset the statistics logged in logStatistics().
magjeddf494b02016-10-07 05:32:35 -0700271 */
magjed9ab8a182016-10-20 03:18:09 -0700272 private void resetStatistics(long currentTimeNs) {
magjeddf494b02016-10-07 05:32:35 -0700273 synchronized (statisticsLock) {
magjed9ab8a182016-10-20 03:18:09 -0700274 statisticsStartTimeNs = currentTimeNs;
magjeddf494b02016-10-07 05:32:35 -0700275 framesReceived = 0;
276 framesDropped = 0;
277 framesRendered = 0;
magjeddf494b02016-10-07 05:32:35 -0700278 renderTimeNs = 0;
magjed9ab8a182016-10-20 03:18:09 -0700279 renderSwapBufferTimeNs = 0;
280 }
281 }
282
283 public void printStackTrace() {
284 synchronized (handlerLock) {
285 final Thread renderThread =
286 (renderThreadHandler == null) ? null : renderThreadHandler.getLooper().getThread();
287 if (renderThread != null) {
288 final StackTraceElement[] renderStackTrace = renderThread.getStackTrace();
289 if (renderStackTrace.length > 0) {
290 logD("EglRenderer stack trace:");
291 for (StackTraceElement traceElem : renderStackTrace) {
292 logD(traceElem.toString());
293 }
294 }
295 }
magjeddf494b02016-10-07 05:32:35 -0700296 }
297 }
298
299 /**
300 * Set if the video stream should be mirrored or not.
301 */
302 public void setMirror(final boolean mirror) {
303 logD("setMirror: " + mirror);
304 synchronized (layoutLock) {
305 this.mirror = mirror;
306 }
307 }
308
309 /**
310 * Set layout aspect ratio. This is used to crop frames when rendering to avoid stretched video.
311 * Set this to 0 to disable cropping.
312 */
313 public void setLayoutAspectRatio(float layoutAspectRatio) {
314 logD("setLayoutAspectRatio: " + layoutAspectRatio);
315 synchronized (layoutLock) {
316 this.layoutAspectRatio = layoutAspectRatio;
317 }
318 }
319
magjed9ab8a182016-10-20 03:18:09 -0700320 /**
321 * Limit render framerate.
322 *
323 * @param fps Limit render framerate to this value, or use Float.POSITIVE_INFINITY to disable fps
324 * reduction.
325 */
326 public void setFpsReduction(float fps) {
327 logD("setFpsReduction: " + fps);
328 synchronized (fpsReductionLock) {
329 final long previousRenderPeriodNs = minRenderPeriodNs;
330 if (fps <= 0) {
331 minRenderPeriodNs = Long.MAX_VALUE;
332 } else {
333 minRenderPeriodNs = (long) (TimeUnit.SECONDS.toNanos(1) / fps);
334 }
335 if (minRenderPeriodNs != previousRenderPeriodNs) {
336 // Fps reduction changed - reset frame time.
337 nextFrameTimeNs = System.nanoTime();
338 }
339 }
340 }
341
342 public void disableFpsReduction() {
343 setFpsReduction(Float.POSITIVE_INFINITY /* fps */);
344 }
345
346 public void pauseVideo() {
347 setFpsReduction(0 /* fps */);
348 }
349
sakalfb0c5732016-11-03 09:15:34 -0700350 /**
sakal3a9bc172016-11-30 08:30:05 -0800351 * Register a callback to be invoked when a new video frame has been received. This version uses
352 * the drawer of the EglRenderer that was passed in init.
sakalfb0c5732016-11-03 09:15:34 -0700353 *
sakald1516522017-03-13 05:11:48 -0700354 * @param listener The callback to be invoked. The callback will be invoked on the render thread.
355 * It should be lightweight and must not call removeFrameListener.
sakalfb0c5732016-11-03 09:15:34 -0700356 * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is
357 * required.
358 */
sakalbb584352016-11-28 08:53:44 -0800359 public void addFrameListener(final FrameListener listener, final float scale) {
sakal8fdf9572017-05-31 02:43:10 -0700360 addFrameListener(listener, scale, null, false /* applyFpsReduction */);
sakal3a9bc172016-11-30 08:30:05 -0800361 }
362
363 /**
364 * Register a callback to be invoked when a new video frame has been received.
365 *
sakald1516522017-03-13 05:11:48 -0700366 * @param listener The callback to be invoked. The callback will be invoked on the render thread.
367 * It should be lightweight and must not call removeFrameListener.
sakal3a9bc172016-11-30 08:30:05 -0800368 * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is
369 * required.
sakald1516522017-03-13 05:11:48 -0700370 * @param drawer Custom drawer to use for this frame listener or null to use the default one.
sakal3a9bc172016-11-30 08:30:05 -0800371 */
372 public void addFrameListener(
sakald1516522017-03-13 05:11:48 -0700373 final FrameListener listener, final float scale, final RendererCommon.GlDrawer drawerParam) {
sakal8fdf9572017-05-31 02:43:10 -0700374 addFrameListener(listener, scale, drawerParam, false /* applyFpsReduction */);
375 }
376
377 /**
378 * Register a callback to be invoked when a new video frame has been received.
379 *
380 * @param listener The callback to be invoked. The callback will be invoked on the render thread.
381 * It should be lightweight and must not call removeFrameListener.
382 * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is
383 * required.
384 * @param drawer Custom drawer to use for this frame listener or null to use the default one.
385 * @param applyFpsReduction This callback will not be called for frames that have been dropped by
386 * FPS reduction.
387 */
388 public void addFrameListener(final FrameListener listener, final float scale,
Sami Kalliomäkie7592d82018-03-22 13:32:44 +0100389 @Nullable final RendererCommon.GlDrawer drawerParam, final boolean applyFpsReduction) {
sakalbf080602017-08-11 01:42:43 -0700390 postToRenderThread(() -> {
391 final RendererCommon.GlDrawer listenerDrawer = drawerParam == null ? drawer : drawerParam;
392 frameListeners.add(
393 new FrameListenerAndParams(listener, scale, listenerDrawer, applyFpsReduction));
sakalbb584352016-11-28 08:53:44 -0800394 });
sakalfb0c5732016-11-03 09:15:34 -0700395 }
396
397 /**
398 * Remove any pending callback that was added with addFrameListener. If the callback is not in
sakalbb584352016-11-28 08:53:44 -0800399 * the queue, nothing happens. It is ensured that callback won't be called after this method
400 * returns.
sakalfb0c5732016-11-03 09:15:34 -0700401 *
402 * @param runnable The callback to remove.
403 */
sakalbb584352016-11-28 08:53:44 -0800404 public void removeFrameListener(final FrameListener listener) {
405 final CountDownLatch latch = new CountDownLatch(1);
Sami Kalliomäki8ebac242017-11-08 17:13:13 +0100406 synchronized (handlerLock) {
407 if (renderThreadHandler == null) {
408 return;
sakalfb0c5732016-11-03 09:15:34 -0700409 }
Sami Kalliomäki8ebac242017-11-08 17:13:13 +0100410 if (Thread.currentThread() == renderThreadHandler.getLooper().getThread()) {
411 throw new RuntimeException("removeFrameListener must not be called on the render thread.");
412 }
413 postToRenderThread(() -> {
414 latch.countDown();
415 final Iterator<FrameListenerAndParams> iter = frameListeners.iterator();
416 while (iter.hasNext()) {
417 if (iter.next().listener == listener) {
418 iter.remove();
419 }
420 }
421 });
422 }
sakalbb584352016-11-28 08:53:44 -0800423 ThreadUtils.awaitUninterruptibly(latch);
sakalfb0c5732016-11-03 09:15:34 -0700424 }
425
magjeddf494b02016-10-07 05:32:35 -0700426 // VideoRenderer.Callbacks interface.
427 @Override
428 public void renderFrame(VideoRenderer.I420Frame frame) {
sakal6bdcefc2017-08-15 01:56:02 -0700429 VideoFrame videoFrame = frame.toVideoFrame();
430 onFrame(videoFrame);
431 videoFrame.release();
432 }
433
434 // VideoSink interface.
435 @Override
436 public void onFrame(VideoFrame frame) {
magjeddf494b02016-10-07 05:32:35 -0700437 synchronized (statisticsLock) {
438 ++framesReceived;
439 }
magjed9ab8a182016-10-20 03:18:09 -0700440 final boolean dropOldFrame;
magjeddf494b02016-10-07 05:32:35 -0700441 synchronized (handlerLock) {
442 if (renderThreadHandler == null) {
443 logD("Dropping frame - Not initialized or already released.");
magjeddf494b02016-10-07 05:32:35 -0700444 return;
445 }
magjed9ab8a182016-10-20 03:18:09 -0700446 synchronized (frameLock) {
447 dropOldFrame = (pendingFrame != null);
448 if (dropOldFrame) {
sakal6bdcefc2017-08-15 01:56:02 -0700449 pendingFrame.release();
magjeddf494b02016-10-07 05:32:35 -0700450 }
451 pendingFrame = frame;
sakal6bdcefc2017-08-15 01:56:02 -0700452 pendingFrame.retain();
sakalbf080602017-08-11 01:42:43 -0700453 renderThreadHandler.post(this ::renderFrameOnRenderThread);
magjeddf494b02016-10-07 05:32:35 -0700454 }
455 }
magjed9ab8a182016-10-20 03:18:09 -0700456 if (dropOldFrame) {
457 synchronized (statisticsLock) {
458 ++framesDropped;
459 }
460 }
magjeddf494b02016-10-07 05:32:35 -0700461 }
462
463 /**
464 * Release EGL surface. This function will block until the EGL surface is released.
465 */
sakal28ec6bd2016-11-09 01:47:12 -0800466 public void releaseEglSurface(final Runnable completionCallback) {
magjeddf494b02016-10-07 05:32:35 -0700467 // Ensure that the render thread is no longer touching the Surface before returning from this
468 // function.
469 eglSurfaceCreationRunnable.setSurface(null /* surface */);
470 synchronized (handlerLock) {
471 if (renderThreadHandler != null) {
472 renderThreadHandler.removeCallbacks(eglSurfaceCreationRunnable);
sakalbf080602017-08-11 01:42:43 -0700473 renderThreadHandler.postAtFrontOfQueue(() -> {
474 if (eglBase != null) {
475 eglBase.detachCurrent();
476 eglBase.releaseSurface();
magjeddf494b02016-10-07 05:32:35 -0700477 }
sakalbf080602017-08-11 01:42:43 -0700478 completionCallback.run();
magjeddf494b02016-10-07 05:32:35 -0700479 });
sakal28ec6bd2016-11-09 01:47:12 -0800480 return;
magjeddf494b02016-10-07 05:32:35 -0700481 }
482 }
sakal28ec6bd2016-11-09 01:47:12 -0800483 completionCallback.run();
magjeddf494b02016-10-07 05:32:35 -0700484 }
485
486 /**
magjeddf494b02016-10-07 05:32:35 -0700487 * Private helper function to post tasks safely.
488 */
magjed9ab8a182016-10-20 03:18:09 -0700489 private void postToRenderThread(Runnable runnable) {
magjeddf494b02016-10-07 05:32:35 -0700490 synchronized (handlerLock) {
491 if (renderThreadHandler != null) {
492 renderThreadHandler.post(runnable);
493 }
494 }
495 }
496
sakalf25a2202017-05-04 06:06:56 -0700497 private void clearSurfaceOnRenderThread(float r, float g, float b, float a) {
magjeddf494b02016-10-07 05:32:35 -0700498 if (eglBase != null && eglBase.hasSurface()) {
499 logD("clearSurface");
sakalf25a2202017-05-04 06:06:56 -0700500 GLES20.glClearColor(r, g, b, a);
magjeddf494b02016-10-07 05:32:35 -0700501 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
502 eglBase.swapBuffers();
503 }
504 }
505
506 /**
sakalf25a2202017-05-04 06:06:56 -0700507 * Post a task to clear the surface to a transparent uniform color.
magjed9ab8a182016-10-20 03:18:09 -0700508 */
509 public void clearImage() {
sakalf25a2202017-05-04 06:06:56 -0700510 clearImage(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */);
511 }
512
513 /**
514 * Post a task to clear the surface to a specific color.
515 */
516 public void clearImage(final float r, final float g, final float b, final float a) {
magjed9ab8a182016-10-20 03:18:09 -0700517 synchronized (handlerLock) {
518 if (renderThreadHandler == null) {
519 return;
520 }
sakalbf080602017-08-11 01:42:43 -0700521 renderThreadHandler.postAtFrontOfQueue(() -> clearSurfaceOnRenderThread(r, g, b, a));
magjed9ab8a182016-10-20 03:18:09 -0700522 }
523 }
524
525 /**
magjeddf494b02016-10-07 05:32:35 -0700526 * Renders and releases |pendingFrame|.
527 */
528 private void renderFrameOnRenderThread() {
529 // Fetch and render |pendingFrame|.
sakal6bdcefc2017-08-15 01:56:02 -0700530 final VideoFrame frame;
magjeddf494b02016-10-07 05:32:35 -0700531 synchronized (frameLock) {
532 if (pendingFrame == null) {
533 return;
534 }
535 frame = pendingFrame;
536 pendingFrame = null;
537 }
538 if (eglBase == null || !eglBase.hasSurface()) {
539 logD("Dropping frame - No surface");
sakal6bdcefc2017-08-15 01:56:02 -0700540 frame.release();
magjeddf494b02016-10-07 05:32:35 -0700541 return;
542 }
sakald1516522017-03-13 05:11:48 -0700543 // Check if fps reduction is active.
544 final boolean shouldRenderFrame;
545 synchronized (fpsReductionLock) {
546 if (minRenderPeriodNs == Long.MAX_VALUE) {
547 // Rendering is paused.
548 shouldRenderFrame = false;
549 } else if (minRenderPeriodNs <= 0) {
550 // FPS reduction is disabled.
551 shouldRenderFrame = true;
552 } else {
553 final long currentTimeNs = System.nanoTime();
554 if (currentTimeNs < nextFrameTimeNs) {
555 logD("Skipping frame rendering - fps reduction is active.");
556 shouldRenderFrame = false;
557 } else {
558 nextFrameTimeNs += minRenderPeriodNs;
559 // The time for the next frame should always be in the future.
560 nextFrameTimeNs = Math.max(nextFrameTimeNs, currentTimeNs);
561 shouldRenderFrame = true;
562 }
563 }
564 }
magjeddf494b02016-10-07 05:32:35 -0700565
566 final long startTimeNs = System.nanoTime();
magjeddf494b02016-10-07 05:32:35 -0700567
sakal6bdcefc2017-08-15 01:56:02 -0700568 final float frameAspectRatio = frame.getRotatedWidth() / (float) frame.getRotatedHeight();
569 final float drawnAspectRatio;
magjeddf494b02016-10-07 05:32:35 -0700570 synchronized (layoutLock) {
sakal6bdcefc2017-08-15 01:56:02 -0700571 drawnAspectRatio = layoutAspectRatio != 0f ? layoutAspectRatio : frameAspectRatio;
magjeddf494b02016-10-07 05:32:35 -0700572 }
573
sakal6bdcefc2017-08-15 01:56:02 -0700574 final float scaleX;
575 final float scaleY;
576
577 if (frameAspectRatio > drawnAspectRatio) {
578 scaleX = drawnAspectRatio / frameAspectRatio;
579 scaleY = 1f;
580 } else {
581 scaleX = 1f;
582 scaleY = frameAspectRatio / drawnAspectRatio;
583 }
584
magjed7cede372017-09-11 06:12:07 -0700585 drawMatrix.reset();
sakal6bdcefc2017-08-15 01:56:02 -0700586 drawMatrix.preTranslate(0.5f, 0.5f);
sakal6bdcefc2017-08-15 01:56:02 -0700587 if (mirror)
588 drawMatrix.preScale(-1f, 1f);
589 drawMatrix.preScale(scaleX, scaleY);
590 drawMatrix.preTranslate(-0.5f, -0.5f);
591
sakald1516522017-03-13 05:11:48 -0700592 if (shouldRenderFrame) {
593 GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */);
594 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
magjed7cede372017-09-11 06:12:07 -0700595 frameDrawer.drawFrame(frame, drawer, drawMatrix, 0 /* viewportX */, 0 /* viewportY */,
596 eglBase.surfaceWidth(), eglBase.surfaceHeight());
sakald1516522017-03-13 05:11:48 -0700597
598 final long swapBuffersStartTimeNs = System.nanoTime();
599 eglBase.swapBuffers();
600
601 final long currentTimeNs = System.nanoTime();
602 synchronized (statisticsLock) {
603 ++framesRendered;
604 renderTimeNs += (currentTimeNs - startTimeNs);
605 renderSwapBufferTimeNs += (currentTimeNs - swapBuffersStartTimeNs);
606 }
magjeddf494b02016-10-07 05:32:35 -0700607 }
608
magjed7cede372017-09-11 06:12:07 -0700609 notifyCallbacks(frame, shouldRenderFrame);
610 frame.release();
sakalfb0c5732016-11-03 09:15:34 -0700611 }
612
magjed7cede372017-09-11 06:12:07 -0700613 private void notifyCallbacks(VideoFrame frame, boolean wasRendered) {
sakalbb584352016-11-28 08:53:44 -0800614 if (frameListeners.isEmpty())
615 return;
sakalfb0c5732016-11-03 09:15:34 -0700616
magjed7cede372017-09-11 06:12:07 -0700617 drawMatrix.reset();
sakal6bdcefc2017-08-15 01:56:02 -0700618 drawMatrix.preTranslate(0.5f, 0.5f);
sakal6bdcefc2017-08-15 01:56:02 -0700619 if (mirror)
620 drawMatrix.preScale(-1f, 1f);
621 drawMatrix.preScale(1f, -1f); // We want the output to be upside down for Bitmap.
622 drawMatrix.preTranslate(-0.5f, -0.5f);
sakalfb0c5732016-11-03 09:15:34 -0700623
sakal8fdf9572017-05-31 02:43:10 -0700624 Iterator<FrameListenerAndParams> it = frameListeners.iterator();
625 while (it.hasNext()) {
626 FrameListenerAndParams listenerAndParams = it.next();
627 if (!wasRendered && listenerAndParams.applyFpsReduction) {
628 continue;
629 }
630 it.remove();
631
sakal6bdcefc2017-08-15 01:56:02 -0700632 final int scaledWidth = (int) (listenerAndParams.scale * frame.getRotatedWidth());
633 final int scaledHeight = (int) (listenerAndParams.scale * frame.getRotatedHeight());
sakalfb0c5732016-11-03 09:15:34 -0700634
635 if (scaledWidth == 0 || scaledHeight == 0) {
sakal3a9bc172016-11-30 08:30:05 -0800636 listenerAndParams.listener.onFrame(null);
sakalfb0c5732016-11-03 09:15:34 -0700637 continue;
638 }
639
640 if (bitmapTextureFramebuffer == null) {
641 bitmapTextureFramebuffer = new GlTextureFrameBuffer(GLES20.GL_RGBA);
642 }
643 bitmapTextureFramebuffer.setSize(scaledWidth, scaledHeight);
644
645 GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, bitmapTextureFramebuffer.getFrameBufferId());
646 GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0,
647 GLES20.GL_TEXTURE_2D, bitmapTextureFramebuffer.getTextureId(), 0);
648
sakal103988d2017-02-17 09:59:01 -0800649 GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */);
650 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
magjed7cede372017-09-11 06:12:07 -0700651 frameDrawer.drawFrame(frame, listenerAndParams.drawer, drawMatrix, 0 /* viewportX */,
652 0 /* viewportY */, scaledWidth, scaledHeight);
sakalfb0c5732016-11-03 09:15:34 -0700653
654 final ByteBuffer bitmapBuffer = ByteBuffer.allocateDirect(scaledWidth * scaledHeight * 4);
655 GLES20.glViewport(0, 0, scaledWidth, scaledHeight);
656 GLES20.glReadPixels(
657 0, 0, scaledWidth, scaledHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, bitmapBuffer);
658
659 GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
660 GlUtil.checkNoGLES2Error("EglRenderer.notifyCallbacks");
661
662 final Bitmap bitmap = Bitmap.createBitmap(scaledWidth, scaledHeight, Bitmap.Config.ARGB_8888);
663 bitmap.copyPixelsFromBuffer(bitmapBuffer);
sakal3a9bc172016-11-30 08:30:05 -0800664 listenerAndParams.listener.onFrame(bitmap);
sakalfb0c5732016-11-03 09:15:34 -0700665 }
magjeddf494b02016-10-07 05:32:35 -0700666 }
667
magjed9ab8a182016-10-20 03:18:09 -0700668 private String averageTimeAsString(long sumTimeNs, int count) {
669 return (count <= 0) ? "NA" : TimeUnit.NANOSECONDS.toMicros(sumTimeNs / count) + " μs";
670 }
671
magjeddf494b02016-10-07 05:32:35 -0700672 private void logStatistics() {
magjed9ab8a182016-10-20 03:18:09 -0700673 final long currentTimeNs = System.nanoTime();
magjeddf494b02016-10-07 05:32:35 -0700674 synchronized (statisticsLock) {
magjed9ab8a182016-10-20 03:18:09 -0700675 final long elapsedTimeNs = currentTimeNs - statisticsStartTimeNs;
676 if (elapsedTimeNs <= 0) {
677 return;
magjeddf494b02016-10-07 05:32:35 -0700678 }
magjed9ab8a182016-10-20 03:18:09 -0700679 final float renderFps = framesRendered * TimeUnit.SECONDS.toNanos(1) / (float) elapsedTimeNs;
680 logD("Duration: " + TimeUnit.NANOSECONDS.toMillis(elapsedTimeNs) + " ms."
681 + " Frames received: " + framesReceived + "."
682 + " Dropped: " + framesDropped + "."
683 + " Rendered: " + framesRendered + "."
sakal037b93a2017-01-16 04:57:32 -0800684 + " Render fps: " + String.format(Locale.US, "%.1f", renderFps) + "."
magjed9ab8a182016-10-20 03:18:09 -0700685 + " Average render time: " + averageTimeAsString(renderTimeNs, framesRendered) + "."
686 + " Average swapBuffer time: "
687 + averageTimeAsString(renderSwapBufferTimeNs, framesRendered) + ".");
688 resetStatistics(currentTimeNs);
magjeddf494b02016-10-07 05:32:35 -0700689 }
690 }
691
692 private void logD(String string) {
693 Logging.d(TAG, name + string);
694 }
695}