blob: 8eb1928cef42c338779132716cc6e383cb40dc68 [file] [log] [blame]
magjeddf494b02016-10-07 05:32:35 -07001/*
2 * Copyright 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11package org.webrtc;
12
sakalfb0c5732016-11-03 09:15:34 -070013import android.graphics.Bitmap;
sakal6bdcefc2017-08-15 01:56:02 -070014import android.graphics.Matrix;
magjed9ab8a182016-10-20 03:18:09 -070015import android.graphics.SurfaceTexture;
magjeddf494b02016-10-07 05:32:35 -070016import android.opengl.GLES20;
17import android.os.Handler;
18import android.os.HandlerThread;
19import android.os.Looper;
20import android.view.Surface;
sakalfb0c5732016-11-03 09:15:34 -070021import java.nio.ByteBuffer;
22import java.util.ArrayList;
23import java.util.Iterator;
sakal037b93a2017-01-16 04:57:32 -080024import java.util.Locale;
magjeddf494b02016-10-07 05:32:35 -070025import java.util.concurrent.CountDownLatch;
26import java.util.concurrent.TimeUnit;
27
28/**
29 * Implements org.webrtc.VideoRenderer.Callbacks by displaying the video stream on an EGL Surface.
30 * This class is intended to be used as a helper class for rendering on SurfaceViews and
31 * TextureViews.
32 */
sakal6bdcefc2017-08-15 01:56:02 -070033public class EglRenderer implements VideoRenderer.Callbacks, VideoSink {
magjeddf494b02016-10-07 05:32:35 -070034 private static final String TAG = "EglRenderer";
magjed9ab8a182016-10-20 03:18:09 -070035 private static final long LOG_INTERVAL_SEC = 4;
magjeddf494b02016-10-07 05:32:35 -070036 private static final int MAX_SURFACE_CLEAR_COUNT = 3;
37
sakalfb0c5732016-11-03 09:15:34 -070038 public interface FrameListener { void onFrame(Bitmap frame); }
39
sakal3a9bc172016-11-30 08:30:05 -080040 private static class FrameListenerAndParams {
sakalfb0c5732016-11-03 09:15:34 -070041 public final FrameListener listener;
sakal3a9bc172016-11-30 08:30:05 -080042 public final float scale;
43 public final RendererCommon.GlDrawer drawer;
sakal8fdf9572017-05-31 02:43:10 -070044 public final boolean applyFpsReduction;
sakalfb0c5732016-11-03 09:15:34 -070045
sakal8fdf9572017-05-31 02:43:10 -070046 public FrameListenerAndParams(FrameListener listener, float scale,
47 RendererCommon.GlDrawer drawer, boolean applyFpsReduction) {
sakalfb0c5732016-11-03 09:15:34 -070048 this.listener = listener;
sakal3a9bc172016-11-30 08:30:05 -080049 this.scale = scale;
50 this.drawer = drawer;
sakal8fdf9572017-05-31 02:43:10 -070051 this.applyFpsReduction = applyFpsReduction;
sakalfb0c5732016-11-03 09:15:34 -070052 }
53 }
54
magjeddf494b02016-10-07 05:32:35 -070055 private class EglSurfaceCreation implements Runnable {
magjed9ab8a182016-10-20 03:18:09 -070056 private Object surface;
magjeddf494b02016-10-07 05:32:35 -070057
Mirko Bonadei12251b62017-11-05 19:35:31 -080058 // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression.
59 @SuppressWarnings("NoSynchronizedMethodCheck")
magjed9ab8a182016-10-20 03:18:09 -070060 public synchronized void setSurface(Object surface) {
magjeddf494b02016-10-07 05:32:35 -070061 this.surface = surface;
62 }
63
64 @Override
Mirko Bonadei12251b62017-11-05 19:35:31 -080065 // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression.
66 @SuppressWarnings("NoSynchronizedMethodCheck")
magjeddf494b02016-10-07 05:32:35 -070067 public synchronized void run() {
68 if (surface != null && eglBase != null && !eglBase.hasSurface()) {
magjed9ab8a182016-10-20 03:18:09 -070069 if (surface instanceof Surface) {
70 eglBase.createSurface((Surface) surface);
71 } else if (surface instanceof SurfaceTexture) {
72 eglBase.createSurface((SurfaceTexture) surface);
73 } else {
74 throw new IllegalStateException("Invalid surface: " + surface);
75 }
magjeddf494b02016-10-07 05:32:35 -070076 eglBase.makeCurrent();
77 // Necessary for YUV frames with odd width.
78 GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1);
79 }
80 }
81 }
82
Xiaolei Yu149533a2017-11-03 07:55:01 +080083 protected final String name;
magjeddf494b02016-10-07 05:32:35 -070084
85 // |renderThreadHandler| is a handler for communicating with |renderThread|, and is synchronized
86 // on |handlerLock|.
87 private final Object handlerLock = new Object();
88 private Handler renderThreadHandler;
89
sakal3a9bc172016-11-30 08:30:05 -080090 private final ArrayList<FrameListenerAndParams> frameListeners = new ArrayList<>();
sakalfb0c5732016-11-03 09:15:34 -070091
magjed9ab8a182016-10-20 03:18:09 -070092 // Variables for fps reduction.
93 private final Object fpsReductionLock = new Object();
94 // Time for when next frame should be rendered.
95 private long nextFrameTimeNs;
96 // Minimum duration between frames when fps reduction is active, or -1 if video is completely
97 // paused.
98 private long minRenderPeriodNs;
99
magjeddf494b02016-10-07 05:32:35 -0700100 // EGL and GL resources for drawing YUV/OES textures. After initilization, these are only accessed
101 // from the render thread.
102 private EglBase eglBase;
magjed7cede372017-09-11 06:12:07 -0700103 private final VideoFrameDrawer frameDrawer = new VideoFrameDrawer();
magjeddf494b02016-10-07 05:32:35 -0700104 private RendererCommon.GlDrawer drawer;
magjed7cede372017-09-11 06:12:07 -0700105 private final Matrix drawMatrix = new Matrix();
magjeddf494b02016-10-07 05:32:35 -0700106
107 // Pending frame to render. Serves as a queue with size 1. Synchronized on |frameLock|.
108 private final Object frameLock = new Object();
sakal6bdcefc2017-08-15 01:56:02 -0700109 private VideoFrame pendingFrame;
magjeddf494b02016-10-07 05:32:35 -0700110
111 // These variables are synchronized on |layoutLock|.
112 private final Object layoutLock = new Object();
magjeddf494b02016-10-07 05:32:35 -0700113 private float layoutAspectRatio;
114 // If true, mirrors the video stream horizontally.
115 private boolean mirror;
116
117 // These variables are synchronized on |statisticsLock|.
118 private final Object statisticsLock = new Object();
119 // Total number of video frames received in renderFrame() call.
120 private int framesReceived;
121 // Number of video frames dropped by renderFrame() because previous frame has not been rendered
122 // yet.
123 private int framesDropped;
124 // Number of rendered video frames.
125 private int framesRendered;
magjed9ab8a182016-10-20 03:18:09 -0700126 // Start time for counting these statistics, or 0 if we haven't started measuring yet.
127 private long statisticsStartTimeNs;
magjeddf494b02016-10-07 05:32:35 -0700128 // Time in ns spent in renderFrameOnRenderThread() function.
129 private long renderTimeNs;
magjed9ab8a182016-10-20 03:18:09 -0700130 // Time in ns spent by the render thread in the swapBuffers() function.
131 private long renderSwapBufferTimeNs;
magjeddf494b02016-10-07 05:32:35 -0700132
sakalfb0c5732016-11-03 09:15:34 -0700133 // Used for bitmap capturing.
134 private GlTextureFrameBuffer bitmapTextureFramebuffer;
135
magjed9ab8a182016-10-20 03:18:09 -0700136 private final Runnable logStatisticsRunnable = new Runnable() {
137 @Override
138 public void run() {
139 logStatistics();
140 synchronized (handlerLock) {
141 if (renderThreadHandler != null) {
142 renderThreadHandler.removeCallbacks(logStatisticsRunnable);
143 renderThreadHandler.postDelayed(
144 logStatisticsRunnable, TimeUnit.SECONDS.toMillis(LOG_INTERVAL_SEC));
145 }
146 }
147 }
148 };
149
magjeddf494b02016-10-07 05:32:35 -0700150 private final EglSurfaceCreation eglSurfaceCreationRunnable = new EglSurfaceCreation();
151
152 /**
153 * Standard constructor. The name will be used for the render thread name and included when
154 * logging. In order to render something, you must first call init() and createEglSurface.
155 */
156 public EglRenderer(String name) {
157 this.name = name;
158 }
159
160 /**
161 * Initialize this class, sharing resources with |sharedContext|. The custom |drawer| will be used
162 * for drawing frames on the EGLSurface. This class is responsible for calling release() on
163 * |drawer|. It is allowed to call init() to reinitialize the renderer after a previous
164 * init()/release() cycle.
165 */
166 public void init(final EglBase.Context sharedContext, final int[] configAttributes,
167 RendererCommon.GlDrawer drawer) {
168 synchronized (handlerLock) {
169 if (renderThreadHandler != null) {
170 throw new IllegalStateException(name + "Already initialized");
171 }
172 logD("Initializing EglRenderer");
173 this.drawer = drawer;
174
175 final HandlerThread renderThread = new HandlerThread(name + "EglRenderer");
176 renderThread.start();
177 renderThreadHandler = new Handler(renderThread.getLooper());
178 // Create EGL context on the newly created render thread. It should be possibly to create the
179 // context on this thread and make it current on the render thread, but this causes failure on
180 // some Marvel based JB devices. https://bugs.chromium.org/p/webrtc/issues/detail?id=6350.
sakalbf080602017-08-11 01:42:43 -0700181 ThreadUtils.invokeAtFrontUninterruptibly(renderThreadHandler, () -> {
182 // If sharedContext is null, then texture frames are disabled. This is typically for old
183 // devices that might not be fully spec compliant, so force EGL 1.0 since EGL 1.4 has
184 // caused trouble on some weird devices.
185 if (sharedContext == null) {
186 logD("EglBase10.create context");
187 eglBase = EglBase.createEgl10(configAttributes);
188 } else {
189 logD("EglBase.create shared context");
190 eglBase = EglBase.create(sharedContext, configAttributes);
magjeddf494b02016-10-07 05:32:35 -0700191 }
192 });
magjed9ab8a182016-10-20 03:18:09 -0700193 renderThreadHandler.post(eglSurfaceCreationRunnable);
194 final long currentTimeNs = System.nanoTime();
195 resetStatistics(currentTimeNs);
196 renderThreadHandler.postDelayed(
197 logStatisticsRunnable, TimeUnit.SECONDS.toMillis(LOG_INTERVAL_SEC));
magjeddf494b02016-10-07 05:32:35 -0700198 }
199 }
200
201 public void createEglSurface(Surface surface) {
magjed9ab8a182016-10-20 03:18:09 -0700202 createEglSurfaceInternal(surface);
203 }
204
205 public void createEglSurface(SurfaceTexture surfaceTexture) {
206 createEglSurfaceInternal(surfaceTexture);
207 }
208
209 private void createEglSurfaceInternal(Object surface) {
magjeddf494b02016-10-07 05:32:35 -0700210 eglSurfaceCreationRunnable.setSurface(surface);
magjed9ab8a182016-10-20 03:18:09 -0700211 postToRenderThread(eglSurfaceCreationRunnable);
magjeddf494b02016-10-07 05:32:35 -0700212 }
213
214 /**
215 * Block until any pending frame is returned and all GL resources released, even if an interrupt
216 * occurs. If an interrupt occurs during release(), the interrupt flag will be set. This function
217 * should be called before the Activity is destroyed and the EGLContext is still valid. If you
218 * don't call this function, the GL resources might leak.
219 */
220 public void release() {
magjed9ab8a182016-10-20 03:18:09 -0700221 logD("Releasing.");
magjeddf494b02016-10-07 05:32:35 -0700222 final CountDownLatch eglCleanupBarrier = new CountDownLatch(1);
223 synchronized (handlerLock) {
224 if (renderThreadHandler == null) {
225 logD("Already released");
226 return;
227 }
magjed9ab8a182016-10-20 03:18:09 -0700228 renderThreadHandler.removeCallbacks(logStatisticsRunnable);
magjeddf494b02016-10-07 05:32:35 -0700229 // Release EGL and GL resources on render thread.
sakalbf080602017-08-11 01:42:43 -0700230 renderThreadHandler.postAtFrontOfQueue(() -> {
231 if (drawer != null) {
232 drawer.release();
233 drawer = null;
magjeddf494b02016-10-07 05:32:35 -0700234 }
magjed7cede372017-09-11 06:12:07 -0700235 frameDrawer.release();
sakalbf080602017-08-11 01:42:43 -0700236 if (bitmapTextureFramebuffer != null) {
237 bitmapTextureFramebuffer.release();
238 bitmapTextureFramebuffer = null;
239 }
240 if (eglBase != null) {
241 logD("eglBase detach and release.");
242 eglBase.detachCurrent();
243 eglBase.release();
244 eglBase = null;
245 }
246 eglCleanupBarrier.countDown();
magjeddf494b02016-10-07 05:32:35 -0700247 });
248 final Looper renderLooper = renderThreadHandler.getLooper();
249 // TODO(magjed): Replace this post() with renderLooper.quitSafely() when API support >= 18.
sakalbf080602017-08-11 01:42:43 -0700250 renderThreadHandler.post(() -> {
251 logD("Quitting render thread.");
252 renderLooper.quit();
magjeddf494b02016-10-07 05:32:35 -0700253 });
254 // Don't accept any more frames or messages to the render thread.
255 renderThreadHandler = null;
256 }
257 // Make sure the EGL/GL cleanup posted above is executed.
258 ThreadUtils.awaitUninterruptibly(eglCleanupBarrier);
259 synchronized (frameLock) {
260 if (pendingFrame != null) {
sakal6bdcefc2017-08-15 01:56:02 -0700261 pendingFrame.release();
magjeddf494b02016-10-07 05:32:35 -0700262 pendingFrame = null;
263 }
264 }
magjeddf494b02016-10-07 05:32:35 -0700265 logD("Releasing done.");
266 }
267
268 /**
magjed9ab8a182016-10-20 03:18:09 -0700269 * Reset the statistics logged in logStatistics().
magjeddf494b02016-10-07 05:32:35 -0700270 */
magjed9ab8a182016-10-20 03:18:09 -0700271 private void resetStatistics(long currentTimeNs) {
magjeddf494b02016-10-07 05:32:35 -0700272 synchronized (statisticsLock) {
magjed9ab8a182016-10-20 03:18:09 -0700273 statisticsStartTimeNs = currentTimeNs;
magjeddf494b02016-10-07 05:32:35 -0700274 framesReceived = 0;
275 framesDropped = 0;
276 framesRendered = 0;
magjeddf494b02016-10-07 05:32:35 -0700277 renderTimeNs = 0;
magjed9ab8a182016-10-20 03:18:09 -0700278 renderSwapBufferTimeNs = 0;
279 }
280 }
281
282 public void printStackTrace() {
283 synchronized (handlerLock) {
284 final Thread renderThread =
285 (renderThreadHandler == null) ? null : renderThreadHandler.getLooper().getThread();
286 if (renderThread != null) {
287 final StackTraceElement[] renderStackTrace = renderThread.getStackTrace();
288 if (renderStackTrace.length > 0) {
289 logD("EglRenderer stack trace:");
290 for (StackTraceElement traceElem : renderStackTrace) {
291 logD(traceElem.toString());
292 }
293 }
294 }
magjeddf494b02016-10-07 05:32:35 -0700295 }
296 }
297
298 /**
299 * Set if the video stream should be mirrored or not.
300 */
301 public void setMirror(final boolean mirror) {
302 logD("setMirror: " + mirror);
303 synchronized (layoutLock) {
304 this.mirror = mirror;
305 }
306 }
307
308 /**
309 * Set layout aspect ratio. This is used to crop frames when rendering to avoid stretched video.
310 * Set this to 0 to disable cropping.
311 */
312 public void setLayoutAspectRatio(float layoutAspectRatio) {
313 logD("setLayoutAspectRatio: " + layoutAspectRatio);
314 synchronized (layoutLock) {
315 this.layoutAspectRatio = layoutAspectRatio;
316 }
317 }
318
magjed9ab8a182016-10-20 03:18:09 -0700319 /**
320 * Limit render framerate.
321 *
322 * @param fps Limit render framerate to this value, or use Float.POSITIVE_INFINITY to disable fps
323 * reduction.
324 */
325 public void setFpsReduction(float fps) {
326 logD("setFpsReduction: " + fps);
327 synchronized (fpsReductionLock) {
328 final long previousRenderPeriodNs = minRenderPeriodNs;
329 if (fps <= 0) {
330 minRenderPeriodNs = Long.MAX_VALUE;
331 } else {
332 minRenderPeriodNs = (long) (TimeUnit.SECONDS.toNanos(1) / fps);
333 }
334 if (minRenderPeriodNs != previousRenderPeriodNs) {
335 // Fps reduction changed - reset frame time.
336 nextFrameTimeNs = System.nanoTime();
337 }
338 }
339 }
340
341 public void disableFpsReduction() {
342 setFpsReduction(Float.POSITIVE_INFINITY /* fps */);
343 }
344
345 public void pauseVideo() {
346 setFpsReduction(0 /* fps */);
347 }
348
sakalfb0c5732016-11-03 09:15:34 -0700349 /**
sakal3a9bc172016-11-30 08:30:05 -0800350 * Register a callback to be invoked when a new video frame has been received. This version uses
351 * the drawer of the EglRenderer that was passed in init.
sakalfb0c5732016-11-03 09:15:34 -0700352 *
sakald1516522017-03-13 05:11:48 -0700353 * @param listener The callback to be invoked. The callback will be invoked on the render thread.
354 * It should be lightweight and must not call removeFrameListener.
sakalfb0c5732016-11-03 09:15:34 -0700355 * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is
356 * required.
357 */
sakalbb584352016-11-28 08:53:44 -0800358 public void addFrameListener(final FrameListener listener, final float scale) {
sakal8fdf9572017-05-31 02:43:10 -0700359 addFrameListener(listener, scale, null, false /* applyFpsReduction */);
sakal3a9bc172016-11-30 08:30:05 -0800360 }
361
362 /**
363 * Register a callback to be invoked when a new video frame has been received.
364 *
sakald1516522017-03-13 05:11:48 -0700365 * @param listener The callback to be invoked. The callback will be invoked on the render thread.
366 * It should be lightweight and must not call removeFrameListener.
sakal3a9bc172016-11-30 08:30:05 -0800367 * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is
368 * required.
sakald1516522017-03-13 05:11:48 -0700369 * @param drawer Custom drawer to use for this frame listener or null to use the default one.
sakal3a9bc172016-11-30 08:30:05 -0800370 */
371 public void addFrameListener(
sakald1516522017-03-13 05:11:48 -0700372 final FrameListener listener, final float scale, final RendererCommon.GlDrawer drawerParam) {
sakal8fdf9572017-05-31 02:43:10 -0700373 addFrameListener(listener, scale, drawerParam, false /* applyFpsReduction */);
374 }
375
376 /**
377 * Register a callback to be invoked when a new video frame has been received.
378 *
379 * @param listener The callback to be invoked. The callback will be invoked on the render thread.
380 * It should be lightweight and must not call removeFrameListener.
381 * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is
382 * required.
383 * @param drawer Custom drawer to use for this frame listener or null to use the default one.
384 * @param applyFpsReduction This callback will not be called for frames that have been dropped by
385 * FPS reduction.
386 */
387 public void addFrameListener(final FrameListener listener, final float scale,
388 final RendererCommon.GlDrawer drawerParam, final boolean applyFpsReduction) {
sakalbf080602017-08-11 01:42:43 -0700389 postToRenderThread(() -> {
390 final RendererCommon.GlDrawer listenerDrawer = drawerParam == null ? drawer : drawerParam;
391 frameListeners.add(
392 new FrameListenerAndParams(listener, scale, listenerDrawer, applyFpsReduction));
sakalbb584352016-11-28 08:53:44 -0800393 });
sakalfb0c5732016-11-03 09:15:34 -0700394 }
395
396 /**
397 * Remove any pending callback that was added with addFrameListener. If the callback is not in
sakalbb584352016-11-28 08:53:44 -0800398 * the queue, nothing happens. It is ensured that callback won't be called after this method
399 * returns.
sakalfb0c5732016-11-03 09:15:34 -0700400 *
401 * @param runnable The callback to remove.
402 */
sakalbb584352016-11-28 08:53:44 -0800403 public void removeFrameListener(final FrameListener listener) {
sakald1516522017-03-13 05:11:48 -0700404 if (Thread.currentThread() == renderThreadHandler.getLooper().getThread()) {
405 throw new RuntimeException("removeFrameListener must not be called on the render thread.");
406 }
sakalbb584352016-11-28 08:53:44 -0800407 final CountDownLatch latch = new CountDownLatch(1);
sakalbf080602017-08-11 01:42:43 -0700408 postToRenderThread(() -> {
409 latch.countDown();
410 final Iterator<FrameListenerAndParams> iter = frameListeners.iterator();
411 while (iter.hasNext()) {
412 if (iter.next().listener == listener) {
413 iter.remove();
sakalfb0c5732016-11-03 09:15:34 -0700414 }
415 }
sakalbb584352016-11-28 08:53:44 -0800416 });
417 ThreadUtils.awaitUninterruptibly(latch);
sakalfb0c5732016-11-03 09:15:34 -0700418 }
419
magjeddf494b02016-10-07 05:32:35 -0700420 // VideoRenderer.Callbacks interface.
421 @Override
422 public void renderFrame(VideoRenderer.I420Frame frame) {
sakal6bdcefc2017-08-15 01:56:02 -0700423 VideoFrame videoFrame = frame.toVideoFrame();
424 onFrame(videoFrame);
425 videoFrame.release();
426 }
427
428 // VideoSink interface.
429 @Override
430 public void onFrame(VideoFrame frame) {
magjeddf494b02016-10-07 05:32:35 -0700431 synchronized (statisticsLock) {
432 ++framesReceived;
433 }
magjed9ab8a182016-10-20 03:18:09 -0700434 final boolean dropOldFrame;
magjeddf494b02016-10-07 05:32:35 -0700435 synchronized (handlerLock) {
436 if (renderThreadHandler == null) {
437 logD("Dropping frame - Not initialized or already released.");
magjeddf494b02016-10-07 05:32:35 -0700438 return;
439 }
magjed9ab8a182016-10-20 03:18:09 -0700440 synchronized (frameLock) {
441 dropOldFrame = (pendingFrame != null);
442 if (dropOldFrame) {
sakal6bdcefc2017-08-15 01:56:02 -0700443 pendingFrame.release();
magjeddf494b02016-10-07 05:32:35 -0700444 }
445 pendingFrame = frame;
sakal6bdcefc2017-08-15 01:56:02 -0700446 pendingFrame.retain();
sakalbf080602017-08-11 01:42:43 -0700447 renderThreadHandler.post(this ::renderFrameOnRenderThread);
magjeddf494b02016-10-07 05:32:35 -0700448 }
449 }
magjed9ab8a182016-10-20 03:18:09 -0700450 if (dropOldFrame) {
451 synchronized (statisticsLock) {
452 ++framesDropped;
453 }
454 }
magjeddf494b02016-10-07 05:32:35 -0700455 }
456
457 /**
458 * Release EGL surface. This function will block until the EGL surface is released.
459 */
sakal28ec6bd2016-11-09 01:47:12 -0800460 public void releaseEglSurface(final Runnable completionCallback) {
magjeddf494b02016-10-07 05:32:35 -0700461 // Ensure that the render thread is no longer touching the Surface before returning from this
462 // function.
463 eglSurfaceCreationRunnable.setSurface(null /* surface */);
464 synchronized (handlerLock) {
465 if (renderThreadHandler != null) {
466 renderThreadHandler.removeCallbacks(eglSurfaceCreationRunnable);
sakalbf080602017-08-11 01:42:43 -0700467 renderThreadHandler.postAtFrontOfQueue(() -> {
468 if (eglBase != null) {
469 eglBase.detachCurrent();
470 eglBase.releaseSurface();
magjeddf494b02016-10-07 05:32:35 -0700471 }
sakalbf080602017-08-11 01:42:43 -0700472 completionCallback.run();
magjeddf494b02016-10-07 05:32:35 -0700473 });
sakal28ec6bd2016-11-09 01:47:12 -0800474 return;
magjeddf494b02016-10-07 05:32:35 -0700475 }
476 }
sakal28ec6bd2016-11-09 01:47:12 -0800477 completionCallback.run();
magjeddf494b02016-10-07 05:32:35 -0700478 }
479
480 /**
magjeddf494b02016-10-07 05:32:35 -0700481 * Private helper function to post tasks safely.
482 */
magjed9ab8a182016-10-20 03:18:09 -0700483 private void postToRenderThread(Runnable runnable) {
magjeddf494b02016-10-07 05:32:35 -0700484 synchronized (handlerLock) {
485 if (renderThreadHandler != null) {
486 renderThreadHandler.post(runnable);
487 }
488 }
489 }
490
sakalf25a2202017-05-04 06:06:56 -0700491 private void clearSurfaceOnRenderThread(float r, float g, float b, float a) {
magjeddf494b02016-10-07 05:32:35 -0700492 if (eglBase != null && eglBase.hasSurface()) {
493 logD("clearSurface");
sakalf25a2202017-05-04 06:06:56 -0700494 GLES20.glClearColor(r, g, b, a);
magjeddf494b02016-10-07 05:32:35 -0700495 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
496 eglBase.swapBuffers();
497 }
498 }
499
500 /**
sakalf25a2202017-05-04 06:06:56 -0700501 * Post a task to clear the surface to a transparent uniform color.
magjed9ab8a182016-10-20 03:18:09 -0700502 */
503 public void clearImage() {
sakalf25a2202017-05-04 06:06:56 -0700504 clearImage(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */);
505 }
506
507 /**
508 * Post a task to clear the surface to a specific color.
509 */
510 public void clearImage(final float r, final float g, final float b, final float a) {
magjed9ab8a182016-10-20 03:18:09 -0700511 synchronized (handlerLock) {
512 if (renderThreadHandler == null) {
513 return;
514 }
sakalbf080602017-08-11 01:42:43 -0700515 renderThreadHandler.postAtFrontOfQueue(() -> clearSurfaceOnRenderThread(r, g, b, a));
magjed9ab8a182016-10-20 03:18:09 -0700516 }
517 }
518
519 /**
magjeddf494b02016-10-07 05:32:35 -0700520 * Renders and releases |pendingFrame|.
521 */
522 private void renderFrameOnRenderThread() {
523 // Fetch and render |pendingFrame|.
sakal6bdcefc2017-08-15 01:56:02 -0700524 final VideoFrame frame;
magjeddf494b02016-10-07 05:32:35 -0700525 synchronized (frameLock) {
526 if (pendingFrame == null) {
527 return;
528 }
529 frame = pendingFrame;
530 pendingFrame = null;
531 }
532 if (eglBase == null || !eglBase.hasSurface()) {
533 logD("Dropping frame - No surface");
sakal6bdcefc2017-08-15 01:56:02 -0700534 frame.release();
magjeddf494b02016-10-07 05:32:35 -0700535 return;
536 }
sakald1516522017-03-13 05:11:48 -0700537 // Check if fps reduction is active.
538 final boolean shouldRenderFrame;
539 synchronized (fpsReductionLock) {
540 if (minRenderPeriodNs == Long.MAX_VALUE) {
541 // Rendering is paused.
542 shouldRenderFrame = false;
543 } else if (minRenderPeriodNs <= 0) {
544 // FPS reduction is disabled.
545 shouldRenderFrame = true;
546 } else {
547 final long currentTimeNs = System.nanoTime();
548 if (currentTimeNs < nextFrameTimeNs) {
549 logD("Skipping frame rendering - fps reduction is active.");
550 shouldRenderFrame = false;
551 } else {
552 nextFrameTimeNs += minRenderPeriodNs;
553 // The time for the next frame should always be in the future.
554 nextFrameTimeNs = Math.max(nextFrameTimeNs, currentTimeNs);
555 shouldRenderFrame = true;
556 }
557 }
558 }
magjeddf494b02016-10-07 05:32:35 -0700559
560 final long startTimeNs = System.nanoTime();
magjeddf494b02016-10-07 05:32:35 -0700561
sakal6bdcefc2017-08-15 01:56:02 -0700562 final float frameAspectRatio = frame.getRotatedWidth() / (float) frame.getRotatedHeight();
563 final float drawnAspectRatio;
magjeddf494b02016-10-07 05:32:35 -0700564 synchronized (layoutLock) {
sakal6bdcefc2017-08-15 01:56:02 -0700565 drawnAspectRatio = layoutAspectRatio != 0f ? layoutAspectRatio : frameAspectRatio;
magjeddf494b02016-10-07 05:32:35 -0700566 }
567
sakal6bdcefc2017-08-15 01:56:02 -0700568 final float scaleX;
569 final float scaleY;
570
571 if (frameAspectRatio > drawnAspectRatio) {
572 scaleX = drawnAspectRatio / frameAspectRatio;
573 scaleY = 1f;
574 } else {
575 scaleX = 1f;
576 scaleY = frameAspectRatio / drawnAspectRatio;
577 }
578
magjed7cede372017-09-11 06:12:07 -0700579 drawMatrix.reset();
sakal6bdcefc2017-08-15 01:56:02 -0700580 drawMatrix.preTranslate(0.5f, 0.5f);
sakal6bdcefc2017-08-15 01:56:02 -0700581 if (mirror)
582 drawMatrix.preScale(-1f, 1f);
583 drawMatrix.preScale(scaleX, scaleY);
584 drawMatrix.preTranslate(-0.5f, -0.5f);
585
sakald1516522017-03-13 05:11:48 -0700586 if (shouldRenderFrame) {
587 GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */);
588 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
magjed7cede372017-09-11 06:12:07 -0700589 frameDrawer.drawFrame(frame, drawer, drawMatrix, 0 /* viewportX */, 0 /* viewportY */,
590 eglBase.surfaceWidth(), eglBase.surfaceHeight());
sakald1516522017-03-13 05:11:48 -0700591
592 final long swapBuffersStartTimeNs = System.nanoTime();
593 eglBase.swapBuffers();
594
595 final long currentTimeNs = System.nanoTime();
596 synchronized (statisticsLock) {
597 ++framesRendered;
598 renderTimeNs += (currentTimeNs - startTimeNs);
599 renderSwapBufferTimeNs += (currentTimeNs - swapBuffersStartTimeNs);
600 }
magjeddf494b02016-10-07 05:32:35 -0700601 }
602
magjed7cede372017-09-11 06:12:07 -0700603 notifyCallbacks(frame, shouldRenderFrame);
604 frame.release();
sakalfb0c5732016-11-03 09:15:34 -0700605 }
606
magjed7cede372017-09-11 06:12:07 -0700607 private void notifyCallbacks(VideoFrame frame, boolean wasRendered) {
sakalbb584352016-11-28 08:53:44 -0800608 if (frameListeners.isEmpty())
609 return;
sakalfb0c5732016-11-03 09:15:34 -0700610
magjed7cede372017-09-11 06:12:07 -0700611 drawMatrix.reset();
sakal6bdcefc2017-08-15 01:56:02 -0700612 drawMatrix.preTranslate(0.5f, 0.5f);
sakal6bdcefc2017-08-15 01:56:02 -0700613 if (mirror)
614 drawMatrix.preScale(-1f, 1f);
615 drawMatrix.preScale(1f, -1f); // We want the output to be upside down for Bitmap.
616 drawMatrix.preTranslate(-0.5f, -0.5f);
sakalfb0c5732016-11-03 09:15:34 -0700617
sakal8fdf9572017-05-31 02:43:10 -0700618 Iterator<FrameListenerAndParams> it = frameListeners.iterator();
619 while (it.hasNext()) {
620 FrameListenerAndParams listenerAndParams = it.next();
621 if (!wasRendered && listenerAndParams.applyFpsReduction) {
622 continue;
623 }
624 it.remove();
625
sakal6bdcefc2017-08-15 01:56:02 -0700626 final int scaledWidth = (int) (listenerAndParams.scale * frame.getRotatedWidth());
627 final int scaledHeight = (int) (listenerAndParams.scale * frame.getRotatedHeight());
sakalfb0c5732016-11-03 09:15:34 -0700628
629 if (scaledWidth == 0 || scaledHeight == 0) {
sakal3a9bc172016-11-30 08:30:05 -0800630 listenerAndParams.listener.onFrame(null);
sakalfb0c5732016-11-03 09:15:34 -0700631 continue;
632 }
633
634 if (bitmapTextureFramebuffer == null) {
635 bitmapTextureFramebuffer = new GlTextureFrameBuffer(GLES20.GL_RGBA);
636 }
637 bitmapTextureFramebuffer.setSize(scaledWidth, scaledHeight);
638
639 GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, bitmapTextureFramebuffer.getFrameBufferId());
640 GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0,
641 GLES20.GL_TEXTURE_2D, bitmapTextureFramebuffer.getTextureId(), 0);
642
sakal103988d2017-02-17 09:59:01 -0800643 GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */);
644 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
magjed7cede372017-09-11 06:12:07 -0700645 frameDrawer.drawFrame(frame, listenerAndParams.drawer, drawMatrix, 0 /* viewportX */,
646 0 /* viewportY */, scaledWidth, scaledHeight);
sakalfb0c5732016-11-03 09:15:34 -0700647
648 final ByteBuffer bitmapBuffer = ByteBuffer.allocateDirect(scaledWidth * scaledHeight * 4);
649 GLES20.glViewport(0, 0, scaledWidth, scaledHeight);
650 GLES20.glReadPixels(
651 0, 0, scaledWidth, scaledHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, bitmapBuffer);
652
653 GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
654 GlUtil.checkNoGLES2Error("EglRenderer.notifyCallbacks");
655
656 final Bitmap bitmap = Bitmap.createBitmap(scaledWidth, scaledHeight, Bitmap.Config.ARGB_8888);
657 bitmap.copyPixelsFromBuffer(bitmapBuffer);
sakal3a9bc172016-11-30 08:30:05 -0800658 listenerAndParams.listener.onFrame(bitmap);
sakalfb0c5732016-11-03 09:15:34 -0700659 }
magjeddf494b02016-10-07 05:32:35 -0700660 }
661
magjed9ab8a182016-10-20 03:18:09 -0700662 private String averageTimeAsString(long sumTimeNs, int count) {
663 return (count <= 0) ? "NA" : TimeUnit.NANOSECONDS.toMicros(sumTimeNs / count) + " μs";
664 }
665
magjeddf494b02016-10-07 05:32:35 -0700666 private void logStatistics() {
magjed9ab8a182016-10-20 03:18:09 -0700667 final long currentTimeNs = System.nanoTime();
magjeddf494b02016-10-07 05:32:35 -0700668 synchronized (statisticsLock) {
magjed9ab8a182016-10-20 03:18:09 -0700669 final long elapsedTimeNs = currentTimeNs - statisticsStartTimeNs;
670 if (elapsedTimeNs <= 0) {
671 return;
magjeddf494b02016-10-07 05:32:35 -0700672 }
magjed9ab8a182016-10-20 03:18:09 -0700673 final float renderFps = framesRendered * TimeUnit.SECONDS.toNanos(1) / (float) elapsedTimeNs;
674 logD("Duration: " + TimeUnit.NANOSECONDS.toMillis(elapsedTimeNs) + " ms."
675 + " Frames received: " + framesReceived + "."
676 + " Dropped: " + framesDropped + "."
677 + " Rendered: " + framesRendered + "."
sakal037b93a2017-01-16 04:57:32 -0800678 + " Render fps: " + String.format(Locale.US, "%.1f", renderFps) + "."
magjed9ab8a182016-10-20 03:18:09 -0700679 + " Average render time: " + averageTimeAsString(renderTimeNs, framesRendered) + "."
680 + " Average swapBuffer time: "
681 + averageTimeAsString(renderSwapBufferTimeNs, framesRendered) + ".");
682 resetStatistics(currentTimeNs);
magjeddf494b02016-10-07 05:32:35 -0700683 }
684 }
685
686 private void logD(String string) {
687 Logging.d(TAG, name + string);
688 }
689}