blob: ba39971e5bd8da9e0004aa018ba9074050e7e6b8 [file] [log] [blame]
magjeddf494b02016-10-07 05:32:35 -07001/*
2 * Copyright 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11package org.webrtc;
12
sakalfb0c5732016-11-03 09:15:34 -070013import android.graphics.Bitmap;
sakal6bdcefc2017-08-15 01:56:02 -070014import android.graphics.Matrix;
magjed9ab8a182016-10-20 03:18:09 -070015import android.graphics.SurfaceTexture;
magjeddf494b02016-10-07 05:32:35 -070016import android.opengl.GLES20;
17import android.os.Handler;
18import android.os.HandlerThread;
19import android.os.Looper;
20import android.view.Surface;
sakalfb0c5732016-11-03 09:15:34 -070021import java.nio.ByteBuffer;
22import java.util.ArrayList;
23import java.util.Iterator;
sakal037b93a2017-01-16 04:57:32 -080024import java.util.Locale;
magjeddf494b02016-10-07 05:32:35 -070025import java.util.concurrent.CountDownLatch;
26import java.util.concurrent.TimeUnit;
27
28/**
29 * Implements org.webrtc.VideoRenderer.Callbacks by displaying the video stream on an EGL Surface.
30 * This class is intended to be used as a helper class for rendering on SurfaceViews and
31 * TextureViews.
32 */
sakal6bdcefc2017-08-15 01:56:02 -070033public class EglRenderer implements VideoRenderer.Callbacks, VideoSink {
magjeddf494b02016-10-07 05:32:35 -070034 private static final String TAG = "EglRenderer";
magjed9ab8a182016-10-20 03:18:09 -070035 private static final long LOG_INTERVAL_SEC = 4;
magjeddf494b02016-10-07 05:32:35 -070036
sakalfb0c5732016-11-03 09:15:34 -070037 public interface FrameListener { void onFrame(Bitmap frame); }
38
sakal3a9bc172016-11-30 08:30:05 -080039 private static class FrameListenerAndParams {
sakalfb0c5732016-11-03 09:15:34 -070040 public final FrameListener listener;
sakal3a9bc172016-11-30 08:30:05 -080041 public final float scale;
42 public final RendererCommon.GlDrawer drawer;
sakal8fdf9572017-05-31 02:43:10 -070043 public final boolean applyFpsReduction;
sakalfb0c5732016-11-03 09:15:34 -070044
sakal8fdf9572017-05-31 02:43:10 -070045 public FrameListenerAndParams(FrameListener listener, float scale,
46 RendererCommon.GlDrawer drawer, boolean applyFpsReduction) {
sakalfb0c5732016-11-03 09:15:34 -070047 this.listener = listener;
sakal3a9bc172016-11-30 08:30:05 -080048 this.scale = scale;
49 this.drawer = drawer;
sakal8fdf9572017-05-31 02:43:10 -070050 this.applyFpsReduction = applyFpsReduction;
sakalfb0c5732016-11-03 09:15:34 -070051 }
52 }
53
magjeddf494b02016-10-07 05:32:35 -070054 private class EglSurfaceCreation implements Runnable {
magjed9ab8a182016-10-20 03:18:09 -070055 private Object surface;
magjeddf494b02016-10-07 05:32:35 -070056
Mirko Bonadei12251b62017-11-05 19:35:31 -080057 // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression.
58 @SuppressWarnings("NoSynchronizedMethodCheck")
magjed9ab8a182016-10-20 03:18:09 -070059 public synchronized void setSurface(Object surface) {
magjeddf494b02016-10-07 05:32:35 -070060 this.surface = surface;
61 }
62
63 @Override
Mirko Bonadei12251b62017-11-05 19:35:31 -080064 // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression.
65 @SuppressWarnings("NoSynchronizedMethodCheck")
magjeddf494b02016-10-07 05:32:35 -070066 public synchronized void run() {
67 if (surface != null && eglBase != null && !eglBase.hasSurface()) {
magjed9ab8a182016-10-20 03:18:09 -070068 if (surface instanceof Surface) {
69 eglBase.createSurface((Surface) surface);
70 } else if (surface instanceof SurfaceTexture) {
71 eglBase.createSurface((SurfaceTexture) surface);
72 } else {
73 throw new IllegalStateException("Invalid surface: " + surface);
74 }
magjeddf494b02016-10-07 05:32:35 -070075 eglBase.makeCurrent();
76 // Necessary for YUV frames with odd width.
77 GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1);
78 }
79 }
80 }
81
Xiaolei Yu149533a2017-11-03 07:55:01 +080082 protected final String name;
magjeddf494b02016-10-07 05:32:35 -070083
84 // |renderThreadHandler| is a handler for communicating with |renderThread|, and is synchronized
85 // on |handlerLock|.
86 private final Object handlerLock = new Object();
87 private Handler renderThreadHandler;
88
sakal3a9bc172016-11-30 08:30:05 -080089 private final ArrayList<FrameListenerAndParams> frameListeners = new ArrayList<>();
sakalfb0c5732016-11-03 09:15:34 -070090
magjed9ab8a182016-10-20 03:18:09 -070091 // Variables for fps reduction.
92 private final Object fpsReductionLock = new Object();
93 // Time for when next frame should be rendered.
94 private long nextFrameTimeNs;
95 // Minimum duration between frames when fps reduction is active, or -1 if video is completely
96 // paused.
97 private long minRenderPeriodNs;
98
magjeddf494b02016-10-07 05:32:35 -070099 // EGL and GL resources for drawing YUV/OES textures. After initilization, these are only accessed
100 // from the render thread.
101 private EglBase eglBase;
magjed7cede372017-09-11 06:12:07 -0700102 private final VideoFrameDrawer frameDrawer = new VideoFrameDrawer();
magjeddf494b02016-10-07 05:32:35 -0700103 private RendererCommon.GlDrawer drawer;
magjed7cede372017-09-11 06:12:07 -0700104 private final Matrix drawMatrix = new Matrix();
magjeddf494b02016-10-07 05:32:35 -0700105
106 // Pending frame to render. Serves as a queue with size 1. Synchronized on |frameLock|.
107 private final Object frameLock = new Object();
sakal6bdcefc2017-08-15 01:56:02 -0700108 private VideoFrame pendingFrame;
magjeddf494b02016-10-07 05:32:35 -0700109
110 // These variables are synchronized on |layoutLock|.
111 private final Object layoutLock = new Object();
magjeddf494b02016-10-07 05:32:35 -0700112 private float layoutAspectRatio;
113 // If true, mirrors the video stream horizontally.
114 private boolean mirror;
115
116 // These variables are synchronized on |statisticsLock|.
117 private final Object statisticsLock = new Object();
118 // Total number of video frames received in renderFrame() call.
119 private int framesReceived;
120 // Number of video frames dropped by renderFrame() because previous frame has not been rendered
121 // yet.
122 private int framesDropped;
123 // Number of rendered video frames.
124 private int framesRendered;
magjed9ab8a182016-10-20 03:18:09 -0700125 // Start time for counting these statistics, or 0 if we haven't started measuring yet.
126 private long statisticsStartTimeNs;
magjeddf494b02016-10-07 05:32:35 -0700127 // Time in ns spent in renderFrameOnRenderThread() function.
128 private long renderTimeNs;
magjed9ab8a182016-10-20 03:18:09 -0700129 // Time in ns spent by the render thread in the swapBuffers() function.
130 private long renderSwapBufferTimeNs;
magjeddf494b02016-10-07 05:32:35 -0700131
sakalfb0c5732016-11-03 09:15:34 -0700132 // Used for bitmap capturing.
133 private GlTextureFrameBuffer bitmapTextureFramebuffer;
134
magjed9ab8a182016-10-20 03:18:09 -0700135 private final Runnable logStatisticsRunnable = new Runnable() {
136 @Override
137 public void run() {
138 logStatistics();
139 synchronized (handlerLock) {
140 if (renderThreadHandler != null) {
141 renderThreadHandler.removeCallbacks(logStatisticsRunnable);
142 renderThreadHandler.postDelayed(
143 logStatisticsRunnable, TimeUnit.SECONDS.toMillis(LOG_INTERVAL_SEC));
144 }
145 }
146 }
147 };
148
magjeddf494b02016-10-07 05:32:35 -0700149 private final EglSurfaceCreation eglSurfaceCreationRunnable = new EglSurfaceCreation();
150
151 /**
152 * Standard constructor. The name will be used for the render thread name and included when
153 * logging. In order to render something, you must first call init() and createEglSurface.
154 */
155 public EglRenderer(String name) {
156 this.name = name;
157 }
158
159 /**
160 * Initialize this class, sharing resources with |sharedContext|. The custom |drawer| will be used
161 * for drawing frames on the EGLSurface. This class is responsible for calling release() on
162 * |drawer|. It is allowed to call init() to reinitialize the renderer after a previous
163 * init()/release() cycle.
164 */
165 public void init(final EglBase.Context sharedContext, final int[] configAttributes,
166 RendererCommon.GlDrawer drawer) {
167 synchronized (handlerLock) {
168 if (renderThreadHandler != null) {
169 throw new IllegalStateException(name + "Already initialized");
170 }
171 logD("Initializing EglRenderer");
172 this.drawer = drawer;
173
174 final HandlerThread renderThread = new HandlerThread(name + "EglRenderer");
175 renderThread.start();
176 renderThreadHandler = new Handler(renderThread.getLooper());
177 // Create EGL context on the newly created render thread. It should be possibly to create the
178 // context on this thread and make it current on the render thread, but this causes failure on
179 // some Marvel based JB devices. https://bugs.chromium.org/p/webrtc/issues/detail?id=6350.
sakalbf080602017-08-11 01:42:43 -0700180 ThreadUtils.invokeAtFrontUninterruptibly(renderThreadHandler, () -> {
181 // If sharedContext is null, then texture frames are disabled. This is typically for old
182 // devices that might not be fully spec compliant, so force EGL 1.0 since EGL 1.4 has
183 // caused trouble on some weird devices.
184 if (sharedContext == null) {
185 logD("EglBase10.create context");
186 eglBase = EglBase.createEgl10(configAttributes);
187 } else {
188 logD("EglBase.create shared context");
189 eglBase = EglBase.create(sharedContext, configAttributes);
magjeddf494b02016-10-07 05:32:35 -0700190 }
191 });
magjed9ab8a182016-10-20 03:18:09 -0700192 renderThreadHandler.post(eglSurfaceCreationRunnable);
193 final long currentTimeNs = System.nanoTime();
194 resetStatistics(currentTimeNs);
195 renderThreadHandler.postDelayed(
196 logStatisticsRunnable, TimeUnit.SECONDS.toMillis(LOG_INTERVAL_SEC));
magjeddf494b02016-10-07 05:32:35 -0700197 }
198 }
199
200 public void createEglSurface(Surface surface) {
magjed9ab8a182016-10-20 03:18:09 -0700201 createEglSurfaceInternal(surface);
202 }
203
204 public void createEglSurface(SurfaceTexture surfaceTexture) {
205 createEglSurfaceInternal(surfaceTexture);
206 }
207
208 private void createEglSurfaceInternal(Object surface) {
magjeddf494b02016-10-07 05:32:35 -0700209 eglSurfaceCreationRunnable.setSurface(surface);
magjed9ab8a182016-10-20 03:18:09 -0700210 postToRenderThread(eglSurfaceCreationRunnable);
magjeddf494b02016-10-07 05:32:35 -0700211 }
212
213 /**
214 * Block until any pending frame is returned and all GL resources released, even if an interrupt
215 * occurs. If an interrupt occurs during release(), the interrupt flag will be set. This function
216 * should be called before the Activity is destroyed and the EGLContext is still valid. If you
217 * don't call this function, the GL resources might leak.
218 */
219 public void release() {
magjed9ab8a182016-10-20 03:18:09 -0700220 logD("Releasing.");
magjeddf494b02016-10-07 05:32:35 -0700221 final CountDownLatch eglCleanupBarrier = new CountDownLatch(1);
222 synchronized (handlerLock) {
223 if (renderThreadHandler == null) {
224 logD("Already released");
225 return;
226 }
magjed9ab8a182016-10-20 03:18:09 -0700227 renderThreadHandler.removeCallbacks(logStatisticsRunnable);
magjeddf494b02016-10-07 05:32:35 -0700228 // Release EGL and GL resources on render thread.
sakalbf080602017-08-11 01:42:43 -0700229 renderThreadHandler.postAtFrontOfQueue(() -> {
230 if (drawer != null) {
231 drawer.release();
232 drawer = null;
magjeddf494b02016-10-07 05:32:35 -0700233 }
magjed7cede372017-09-11 06:12:07 -0700234 frameDrawer.release();
sakalbf080602017-08-11 01:42:43 -0700235 if (bitmapTextureFramebuffer != null) {
236 bitmapTextureFramebuffer.release();
237 bitmapTextureFramebuffer = null;
238 }
239 if (eglBase != null) {
240 logD("eglBase detach and release.");
241 eglBase.detachCurrent();
242 eglBase.release();
243 eglBase = null;
244 }
Sami Kalliomäki8ebac242017-11-08 17:13:13 +0100245 frameListeners.clear();
sakalbf080602017-08-11 01:42:43 -0700246 eglCleanupBarrier.countDown();
magjeddf494b02016-10-07 05:32:35 -0700247 });
248 final Looper renderLooper = renderThreadHandler.getLooper();
249 // TODO(magjed): Replace this post() with renderLooper.quitSafely() when API support >= 18.
sakalbf080602017-08-11 01:42:43 -0700250 renderThreadHandler.post(() -> {
251 logD("Quitting render thread.");
252 renderLooper.quit();
magjeddf494b02016-10-07 05:32:35 -0700253 });
254 // Don't accept any more frames or messages to the render thread.
255 renderThreadHandler = null;
256 }
257 // Make sure the EGL/GL cleanup posted above is executed.
258 ThreadUtils.awaitUninterruptibly(eglCleanupBarrier);
259 synchronized (frameLock) {
260 if (pendingFrame != null) {
sakal6bdcefc2017-08-15 01:56:02 -0700261 pendingFrame.release();
magjeddf494b02016-10-07 05:32:35 -0700262 pendingFrame = null;
263 }
264 }
magjeddf494b02016-10-07 05:32:35 -0700265 logD("Releasing done.");
266 }
267
268 /**
magjed9ab8a182016-10-20 03:18:09 -0700269 * Reset the statistics logged in logStatistics().
magjeddf494b02016-10-07 05:32:35 -0700270 */
magjed9ab8a182016-10-20 03:18:09 -0700271 private void resetStatistics(long currentTimeNs) {
magjeddf494b02016-10-07 05:32:35 -0700272 synchronized (statisticsLock) {
magjed9ab8a182016-10-20 03:18:09 -0700273 statisticsStartTimeNs = currentTimeNs;
magjeddf494b02016-10-07 05:32:35 -0700274 framesReceived = 0;
275 framesDropped = 0;
276 framesRendered = 0;
magjeddf494b02016-10-07 05:32:35 -0700277 renderTimeNs = 0;
magjed9ab8a182016-10-20 03:18:09 -0700278 renderSwapBufferTimeNs = 0;
279 }
280 }
281
282 public void printStackTrace() {
283 synchronized (handlerLock) {
284 final Thread renderThread =
285 (renderThreadHandler == null) ? null : renderThreadHandler.getLooper().getThread();
286 if (renderThread != null) {
287 final StackTraceElement[] renderStackTrace = renderThread.getStackTrace();
288 if (renderStackTrace.length > 0) {
289 logD("EglRenderer stack trace:");
290 for (StackTraceElement traceElem : renderStackTrace) {
291 logD(traceElem.toString());
292 }
293 }
294 }
magjeddf494b02016-10-07 05:32:35 -0700295 }
296 }
297
298 /**
299 * Set if the video stream should be mirrored or not.
300 */
301 public void setMirror(final boolean mirror) {
302 logD("setMirror: " + mirror);
303 synchronized (layoutLock) {
304 this.mirror = mirror;
305 }
306 }
307
308 /**
309 * Set layout aspect ratio. This is used to crop frames when rendering to avoid stretched video.
310 * Set this to 0 to disable cropping.
311 */
312 public void setLayoutAspectRatio(float layoutAspectRatio) {
313 logD("setLayoutAspectRatio: " + layoutAspectRatio);
314 synchronized (layoutLock) {
315 this.layoutAspectRatio = layoutAspectRatio;
316 }
317 }
318
magjed9ab8a182016-10-20 03:18:09 -0700319 /**
320 * Limit render framerate.
321 *
322 * @param fps Limit render framerate to this value, or use Float.POSITIVE_INFINITY to disable fps
323 * reduction.
324 */
325 public void setFpsReduction(float fps) {
326 logD("setFpsReduction: " + fps);
327 synchronized (fpsReductionLock) {
328 final long previousRenderPeriodNs = minRenderPeriodNs;
329 if (fps <= 0) {
330 minRenderPeriodNs = Long.MAX_VALUE;
331 } else {
332 minRenderPeriodNs = (long) (TimeUnit.SECONDS.toNanos(1) / fps);
333 }
334 if (minRenderPeriodNs != previousRenderPeriodNs) {
335 // Fps reduction changed - reset frame time.
336 nextFrameTimeNs = System.nanoTime();
337 }
338 }
339 }
340
341 public void disableFpsReduction() {
342 setFpsReduction(Float.POSITIVE_INFINITY /* fps */);
343 }
344
345 public void pauseVideo() {
346 setFpsReduction(0 /* fps */);
347 }
348
sakalfb0c5732016-11-03 09:15:34 -0700349 /**
sakal3a9bc172016-11-30 08:30:05 -0800350 * Register a callback to be invoked when a new video frame has been received. This version uses
351 * the drawer of the EglRenderer that was passed in init.
sakalfb0c5732016-11-03 09:15:34 -0700352 *
sakald1516522017-03-13 05:11:48 -0700353 * @param listener The callback to be invoked. The callback will be invoked on the render thread.
354 * It should be lightweight and must not call removeFrameListener.
sakalfb0c5732016-11-03 09:15:34 -0700355 * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is
356 * required.
357 */
sakalbb584352016-11-28 08:53:44 -0800358 public void addFrameListener(final FrameListener listener, final float scale) {
sakal8fdf9572017-05-31 02:43:10 -0700359 addFrameListener(listener, scale, null, false /* applyFpsReduction */);
sakal3a9bc172016-11-30 08:30:05 -0800360 }
361
362 /**
363 * Register a callback to be invoked when a new video frame has been received.
364 *
sakald1516522017-03-13 05:11:48 -0700365 * @param listener The callback to be invoked. The callback will be invoked on the render thread.
366 * It should be lightweight and must not call removeFrameListener.
sakal3a9bc172016-11-30 08:30:05 -0800367 * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is
368 * required.
sakald1516522017-03-13 05:11:48 -0700369 * @param drawer Custom drawer to use for this frame listener or null to use the default one.
sakal3a9bc172016-11-30 08:30:05 -0800370 */
371 public void addFrameListener(
sakald1516522017-03-13 05:11:48 -0700372 final FrameListener listener, final float scale, final RendererCommon.GlDrawer drawerParam) {
sakal8fdf9572017-05-31 02:43:10 -0700373 addFrameListener(listener, scale, drawerParam, false /* applyFpsReduction */);
374 }
375
376 /**
377 * Register a callback to be invoked when a new video frame has been received.
378 *
379 * @param listener The callback to be invoked. The callback will be invoked on the render thread.
380 * It should be lightweight and must not call removeFrameListener.
381 * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is
382 * required.
383 * @param drawer Custom drawer to use for this frame listener or null to use the default one.
384 * @param applyFpsReduction This callback will not be called for frames that have been dropped by
385 * FPS reduction.
386 */
387 public void addFrameListener(final FrameListener listener, final float scale,
388 final RendererCommon.GlDrawer drawerParam, final boolean applyFpsReduction) {
sakalbf080602017-08-11 01:42:43 -0700389 postToRenderThread(() -> {
390 final RendererCommon.GlDrawer listenerDrawer = drawerParam == null ? drawer : drawerParam;
391 frameListeners.add(
392 new FrameListenerAndParams(listener, scale, listenerDrawer, applyFpsReduction));
sakalbb584352016-11-28 08:53:44 -0800393 });
sakalfb0c5732016-11-03 09:15:34 -0700394 }
395
396 /**
397 * Remove any pending callback that was added with addFrameListener. If the callback is not in
sakalbb584352016-11-28 08:53:44 -0800398 * the queue, nothing happens. It is ensured that callback won't be called after this method
399 * returns.
sakalfb0c5732016-11-03 09:15:34 -0700400 *
401 * @param runnable The callback to remove.
402 */
sakalbb584352016-11-28 08:53:44 -0800403 public void removeFrameListener(final FrameListener listener) {
404 final CountDownLatch latch = new CountDownLatch(1);
Sami Kalliomäki8ebac242017-11-08 17:13:13 +0100405 synchronized (handlerLock) {
406 if (renderThreadHandler == null) {
407 return;
sakalfb0c5732016-11-03 09:15:34 -0700408 }
Sami Kalliomäki8ebac242017-11-08 17:13:13 +0100409 if (Thread.currentThread() == renderThreadHandler.getLooper().getThread()) {
410 throw new RuntimeException("removeFrameListener must not be called on the render thread.");
411 }
412 postToRenderThread(() -> {
413 latch.countDown();
414 final Iterator<FrameListenerAndParams> iter = frameListeners.iterator();
415 while (iter.hasNext()) {
416 if (iter.next().listener == listener) {
417 iter.remove();
418 }
419 }
420 });
421 }
sakalbb584352016-11-28 08:53:44 -0800422 ThreadUtils.awaitUninterruptibly(latch);
sakalfb0c5732016-11-03 09:15:34 -0700423 }
424
magjeddf494b02016-10-07 05:32:35 -0700425 // VideoRenderer.Callbacks interface.
426 @Override
427 public void renderFrame(VideoRenderer.I420Frame frame) {
sakal6bdcefc2017-08-15 01:56:02 -0700428 VideoFrame videoFrame = frame.toVideoFrame();
429 onFrame(videoFrame);
430 videoFrame.release();
431 }
432
433 // VideoSink interface.
434 @Override
435 public void onFrame(VideoFrame frame) {
magjeddf494b02016-10-07 05:32:35 -0700436 synchronized (statisticsLock) {
437 ++framesReceived;
438 }
magjed9ab8a182016-10-20 03:18:09 -0700439 final boolean dropOldFrame;
magjeddf494b02016-10-07 05:32:35 -0700440 synchronized (handlerLock) {
441 if (renderThreadHandler == null) {
442 logD("Dropping frame - Not initialized or already released.");
magjeddf494b02016-10-07 05:32:35 -0700443 return;
444 }
magjed9ab8a182016-10-20 03:18:09 -0700445 synchronized (frameLock) {
446 dropOldFrame = (pendingFrame != null);
447 if (dropOldFrame) {
sakal6bdcefc2017-08-15 01:56:02 -0700448 pendingFrame.release();
magjeddf494b02016-10-07 05:32:35 -0700449 }
450 pendingFrame = frame;
sakal6bdcefc2017-08-15 01:56:02 -0700451 pendingFrame.retain();
sakalbf080602017-08-11 01:42:43 -0700452 renderThreadHandler.post(this ::renderFrameOnRenderThread);
magjeddf494b02016-10-07 05:32:35 -0700453 }
454 }
magjed9ab8a182016-10-20 03:18:09 -0700455 if (dropOldFrame) {
456 synchronized (statisticsLock) {
457 ++framesDropped;
458 }
459 }
magjeddf494b02016-10-07 05:32:35 -0700460 }
461
462 /**
463 * Release EGL surface. This function will block until the EGL surface is released.
464 */
sakal28ec6bd2016-11-09 01:47:12 -0800465 public void releaseEglSurface(final Runnable completionCallback) {
magjeddf494b02016-10-07 05:32:35 -0700466 // Ensure that the render thread is no longer touching the Surface before returning from this
467 // function.
468 eglSurfaceCreationRunnable.setSurface(null /* surface */);
469 synchronized (handlerLock) {
470 if (renderThreadHandler != null) {
471 renderThreadHandler.removeCallbacks(eglSurfaceCreationRunnable);
sakalbf080602017-08-11 01:42:43 -0700472 renderThreadHandler.postAtFrontOfQueue(() -> {
473 if (eglBase != null) {
474 eglBase.detachCurrent();
475 eglBase.releaseSurface();
magjeddf494b02016-10-07 05:32:35 -0700476 }
sakalbf080602017-08-11 01:42:43 -0700477 completionCallback.run();
magjeddf494b02016-10-07 05:32:35 -0700478 });
sakal28ec6bd2016-11-09 01:47:12 -0800479 return;
magjeddf494b02016-10-07 05:32:35 -0700480 }
481 }
sakal28ec6bd2016-11-09 01:47:12 -0800482 completionCallback.run();
magjeddf494b02016-10-07 05:32:35 -0700483 }
484
485 /**
magjeddf494b02016-10-07 05:32:35 -0700486 * Private helper function to post tasks safely.
487 */
magjed9ab8a182016-10-20 03:18:09 -0700488 private void postToRenderThread(Runnable runnable) {
magjeddf494b02016-10-07 05:32:35 -0700489 synchronized (handlerLock) {
490 if (renderThreadHandler != null) {
491 renderThreadHandler.post(runnable);
492 }
493 }
494 }
495
sakalf25a2202017-05-04 06:06:56 -0700496 private void clearSurfaceOnRenderThread(float r, float g, float b, float a) {
magjeddf494b02016-10-07 05:32:35 -0700497 if (eglBase != null && eglBase.hasSurface()) {
498 logD("clearSurface");
sakalf25a2202017-05-04 06:06:56 -0700499 GLES20.glClearColor(r, g, b, a);
magjeddf494b02016-10-07 05:32:35 -0700500 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
501 eglBase.swapBuffers();
502 }
503 }
504
505 /**
sakalf25a2202017-05-04 06:06:56 -0700506 * Post a task to clear the surface to a transparent uniform color.
magjed9ab8a182016-10-20 03:18:09 -0700507 */
508 public void clearImage() {
sakalf25a2202017-05-04 06:06:56 -0700509 clearImage(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */);
510 }
511
512 /**
513 * Post a task to clear the surface to a specific color.
514 */
515 public void clearImage(final float r, final float g, final float b, final float a) {
magjed9ab8a182016-10-20 03:18:09 -0700516 synchronized (handlerLock) {
517 if (renderThreadHandler == null) {
518 return;
519 }
sakalbf080602017-08-11 01:42:43 -0700520 renderThreadHandler.postAtFrontOfQueue(() -> clearSurfaceOnRenderThread(r, g, b, a));
magjed9ab8a182016-10-20 03:18:09 -0700521 }
522 }
523
524 /**
magjeddf494b02016-10-07 05:32:35 -0700525 * Renders and releases |pendingFrame|.
526 */
527 private void renderFrameOnRenderThread() {
528 // Fetch and render |pendingFrame|.
sakal6bdcefc2017-08-15 01:56:02 -0700529 final VideoFrame frame;
magjeddf494b02016-10-07 05:32:35 -0700530 synchronized (frameLock) {
531 if (pendingFrame == null) {
532 return;
533 }
534 frame = pendingFrame;
535 pendingFrame = null;
536 }
537 if (eglBase == null || !eglBase.hasSurface()) {
538 logD("Dropping frame - No surface");
sakal6bdcefc2017-08-15 01:56:02 -0700539 frame.release();
magjeddf494b02016-10-07 05:32:35 -0700540 return;
541 }
sakald1516522017-03-13 05:11:48 -0700542 // Check if fps reduction is active.
543 final boolean shouldRenderFrame;
544 synchronized (fpsReductionLock) {
545 if (minRenderPeriodNs == Long.MAX_VALUE) {
546 // Rendering is paused.
547 shouldRenderFrame = false;
548 } else if (minRenderPeriodNs <= 0) {
549 // FPS reduction is disabled.
550 shouldRenderFrame = true;
551 } else {
552 final long currentTimeNs = System.nanoTime();
553 if (currentTimeNs < nextFrameTimeNs) {
554 logD("Skipping frame rendering - fps reduction is active.");
555 shouldRenderFrame = false;
556 } else {
557 nextFrameTimeNs += minRenderPeriodNs;
558 // The time for the next frame should always be in the future.
559 nextFrameTimeNs = Math.max(nextFrameTimeNs, currentTimeNs);
560 shouldRenderFrame = true;
561 }
562 }
563 }
magjeddf494b02016-10-07 05:32:35 -0700564
565 final long startTimeNs = System.nanoTime();
magjeddf494b02016-10-07 05:32:35 -0700566
sakal6bdcefc2017-08-15 01:56:02 -0700567 final float frameAspectRatio = frame.getRotatedWidth() / (float) frame.getRotatedHeight();
568 final float drawnAspectRatio;
magjeddf494b02016-10-07 05:32:35 -0700569 synchronized (layoutLock) {
sakal6bdcefc2017-08-15 01:56:02 -0700570 drawnAspectRatio = layoutAspectRatio != 0f ? layoutAspectRatio : frameAspectRatio;
magjeddf494b02016-10-07 05:32:35 -0700571 }
572
sakal6bdcefc2017-08-15 01:56:02 -0700573 final float scaleX;
574 final float scaleY;
575
576 if (frameAspectRatio > drawnAspectRatio) {
577 scaleX = drawnAspectRatio / frameAspectRatio;
578 scaleY = 1f;
579 } else {
580 scaleX = 1f;
581 scaleY = frameAspectRatio / drawnAspectRatio;
582 }
583
magjed7cede372017-09-11 06:12:07 -0700584 drawMatrix.reset();
sakal6bdcefc2017-08-15 01:56:02 -0700585 drawMatrix.preTranslate(0.5f, 0.5f);
sakal6bdcefc2017-08-15 01:56:02 -0700586 if (mirror)
587 drawMatrix.preScale(-1f, 1f);
588 drawMatrix.preScale(scaleX, scaleY);
589 drawMatrix.preTranslate(-0.5f, -0.5f);
590
sakald1516522017-03-13 05:11:48 -0700591 if (shouldRenderFrame) {
592 GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */);
593 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
magjed7cede372017-09-11 06:12:07 -0700594 frameDrawer.drawFrame(frame, drawer, drawMatrix, 0 /* viewportX */, 0 /* viewportY */,
595 eglBase.surfaceWidth(), eglBase.surfaceHeight());
sakald1516522017-03-13 05:11:48 -0700596
597 final long swapBuffersStartTimeNs = System.nanoTime();
598 eglBase.swapBuffers();
599
600 final long currentTimeNs = System.nanoTime();
601 synchronized (statisticsLock) {
602 ++framesRendered;
603 renderTimeNs += (currentTimeNs - startTimeNs);
604 renderSwapBufferTimeNs += (currentTimeNs - swapBuffersStartTimeNs);
605 }
magjeddf494b02016-10-07 05:32:35 -0700606 }
607
magjed7cede372017-09-11 06:12:07 -0700608 notifyCallbacks(frame, shouldRenderFrame);
609 frame.release();
sakalfb0c5732016-11-03 09:15:34 -0700610 }
611
magjed7cede372017-09-11 06:12:07 -0700612 private void notifyCallbacks(VideoFrame frame, boolean wasRendered) {
sakalbb584352016-11-28 08:53:44 -0800613 if (frameListeners.isEmpty())
614 return;
sakalfb0c5732016-11-03 09:15:34 -0700615
magjed7cede372017-09-11 06:12:07 -0700616 drawMatrix.reset();
sakal6bdcefc2017-08-15 01:56:02 -0700617 drawMatrix.preTranslate(0.5f, 0.5f);
sakal6bdcefc2017-08-15 01:56:02 -0700618 if (mirror)
619 drawMatrix.preScale(-1f, 1f);
620 drawMatrix.preScale(1f, -1f); // We want the output to be upside down for Bitmap.
621 drawMatrix.preTranslate(-0.5f, -0.5f);
sakalfb0c5732016-11-03 09:15:34 -0700622
sakal8fdf9572017-05-31 02:43:10 -0700623 Iterator<FrameListenerAndParams> it = frameListeners.iterator();
624 while (it.hasNext()) {
625 FrameListenerAndParams listenerAndParams = it.next();
626 if (!wasRendered && listenerAndParams.applyFpsReduction) {
627 continue;
628 }
629 it.remove();
630
sakal6bdcefc2017-08-15 01:56:02 -0700631 final int scaledWidth = (int) (listenerAndParams.scale * frame.getRotatedWidth());
632 final int scaledHeight = (int) (listenerAndParams.scale * frame.getRotatedHeight());
sakalfb0c5732016-11-03 09:15:34 -0700633
634 if (scaledWidth == 0 || scaledHeight == 0) {
sakal3a9bc172016-11-30 08:30:05 -0800635 listenerAndParams.listener.onFrame(null);
sakalfb0c5732016-11-03 09:15:34 -0700636 continue;
637 }
638
639 if (bitmapTextureFramebuffer == null) {
640 bitmapTextureFramebuffer = new GlTextureFrameBuffer(GLES20.GL_RGBA);
641 }
642 bitmapTextureFramebuffer.setSize(scaledWidth, scaledHeight);
643
644 GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, bitmapTextureFramebuffer.getFrameBufferId());
645 GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0,
646 GLES20.GL_TEXTURE_2D, bitmapTextureFramebuffer.getTextureId(), 0);
647
sakal103988d2017-02-17 09:59:01 -0800648 GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */);
649 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
magjed7cede372017-09-11 06:12:07 -0700650 frameDrawer.drawFrame(frame, listenerAndParams.drawer, drawMatrix, 0 /* viewportX */,
651 0 /* viewportY */, scaledWidth, scaledHeight);
sakalfb0c5732016-11-03 09:15:34 -0700652
653 final ByteBuffer bitmapBuffer = ByteBuffer.allocateDirect(scaledWidth * scaledHeight * 4);
654 GLES20.glViewport(0, 0, scaledWidth, scaledHeight);
655 GLES20.glReadPixels(
656 0, 0, scaledWidth, scaledHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, bitmapBuffer);
657
658 GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
659 GlUtil.checkNoGLES2Error("EglRenderer.notifyCallbacks");
660
661 final Bitmap bitmap = Bitmap.createBitmap(scaledWidth, scaledHeight, Bitmap.Config.ARGB_8888);
662 bitmap.copyPixelsFromBuffer(bitmapBuffer);
sakal3a9bc172016-11-30 08:30:05 -0800663 listenerAndParams.listener.onFrame(bitmap);
sakalfb0c5732016-11-03 09:15:34 -0700664 }
magjeddf494b02016-10-07 05:32:35 -0700665 }
666
magjed9ab8a182016-10-20 03:18:09 -0700667 private String averageTimeAsString(long sumTimeNs, int count) {
668 return (count <= 0) ? "NA" : TimeUnit.NANOSECONDS.toMicros(sumTimeNs / count) + " μs";
669 }
670
magjeddf494b02016-10-07 05:32:35 -0700671 private void logStatistics() {
magjed9ab8a182016-10-20 03:18:09 -0700672 final long currentTimeNs = System.nanoTime();
magjeddf494b02016-10-07 05:32:35 -0700673 synchronized (statisticsLock) {
magjed9ab8a182016-10-20 03:18:09 -0700674 final long elapsedTimeNs = currentTimeNs - statisticsStartTimeNs;
675 if (elapsedTimeNs <= 0) {
676 return;
magjeddf494b02016-10-07 05:32:35 -0700677 }
magjed9ab8a182016-10-20 03:18:09 -0700678 final float renderFps = framesRendered * TimeUnit.SECONDS.toNanos(1) / (float) elapsedTimeNs;
679 logD("Duration: " + TimeUnit.NANOSECONDS.toMillis(elapsedTimeNs) + " ms."
680 + " Frames received: " + framesReceived + "."
681 + " Dropped: " + framesDropped + "."
682 + " Rendered: " + framesRendered + "."
sakal037b93a2017-01-16 04:57:32 -0800683 + " Render fps: " + String.format(Locale.US, "%.1f", renderFps) + "."
magjed9ab8a182016-10-20 03:18:09 -0700684 + " Average render time: " + averageTimeAsString(renderTimeNs, framesRendered) + "."
685 + " Average swapBuffer time: "
686 + averageTimeAsString(renderSwapBufferTimeNs, framesRendered) + ".");
687 resetStatistics(currentTimeNs);
magjeddf494b02016-10-07 05:32:35 -0700688 }
689 }
690
691 private void logD(String string) {
692 Logging.d(TAG, name + string);
693 }
694}