blob: abf2f4a9f4cde9c1b178b9f12da5a98e09bad9a6 [file] [log] [blame]
magjeddf494b02016-10-07 05:32:35 -07001/*
2 * Copyright 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11package org.webrtc;
12
sakalfb0c5732016-11-03 09:15:34 -070013import android.graphics.Bitmap;
sakal6bdcefc2017-08-15 01:56:02 -070014import android.graphics.Matrix;
magjed9ab8a182016-10-20 03:18:09 -070015import android.graphics.SurfaceTexture;
magjeddf494b02016-10-07 05:32:35 -070016import android.opengl.GLES20;
17import android.os.Handler;
18import android.os.HandlerThread;
19import android.os.Looper;
20import android.view.Surface;
sakalfb0c5732016-11-03 09:15:34 -070021import java.nio.ByteBuffer;
22import java.util.ArrayList;
23import java.util.Iterator;
sakal037b93a2017-01-16 04:57:32 -080024import java.util.Locale;
magjeddf494b02016-10-07 05:32:35 -070025import java.util.concurrent.CountDownLatch;
26import java.util.concurrent.TimeUnit;
27
28/**
29 * Implements org.webrtc.VideoRenderer.Callbacks by displaying the video stream on an EGL Surface.
30 * This class is intended to be used as a helper class for rendering on SurfaceViews and
31 * TextureViews.
32 */
sakal6bdcefc2017-08-15 01:56:02 -070033public class EglRenderer implements VideoRenderer.Callbacks, VideoSink {
magjeddf494b02016-10-07 05:32:35 -070034 private static final String TAG = "EglRenderer";
magjed9ab8a182016-10-20 03:18:09 -070035 private static final long LOG_INTERVAL_SEC = 4;
magjeddf494b02016-10-07 05:32:35 -070036
sakalfb0c5732016-11-03 09:15:34 -070037 public interface FrameListener { void onFrame(Bitmap frame); }
38
sakal3a9bc172016-11-30 08:30:05 -080039 private static class FrameListenerAndParams {
sakalfb0c5732016-11-03 09:15:34 -070040 public final FrameListener listener;
sakal3a9bc172016-11-30 08:30:05 -080041 public final float scale;
42 public final RendererCommon.GlDrawer drawer;
sakal8fdf9572017-05-31 02:43:10 -070043 public final boolean applyFpsReduction;
sakalfb0c5732016-11-03 09:15:34 -070044
sakal8fdf9572017-05-31 02:43:10 -070045 public FrameListenerAndParams(FrameListener listener, float scale,
46 RendererCommon.GlDrawer drawer, boolean applyFpsReduction) {
sakalfb0c5732016-11-03 09:15:34 -070047 this.listener = listener;
sakal3a9bc172016-11-30 08:30:05 -080048 this.scale = scale;
49 this.drawer = drawer;
sakal8fdf9572017-05-31 02:43:10 -070050 this.applyFpsReduction = applyFpsReduction;
sakalfb0c5732016-11-03 09:15:34 -070051 }
52 }
53
magjeddf494b02016-10-07 05:32:35 -070054 private class EglSurfaceCreation implements Runnable {
magjed9ab8a182016-10-20 03:18:09 -070055 private Object surface;
magjeddf494b02016-10-07 05:32:35 -070056
Mirko Bonadei12251b62017-11-05 19:35:31 -080057 // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression.
58 @SuppressWarnings("NoSynchronizedMethodCheck")
magjed9ab8a182016-10-20 03:18:09 -070059 public synchronized void setSurface(Object surface) {
magjeddf494b02016-10-07 05:32:35 -070060 this.surface = surface;
61 }
62
63 @Override
Mirko Bonadei12251b62017-11-05 19:35:31 -080064 // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression.
65 @SuppressWarnings("NoSynchronizedMethodCheck")
magjeddf494b02016-10-07 05:32:35 -070066 public synchronized void run() {
67 if (surface != null && eglBase != null && !eglBase.hasSurface()) {
magjed9ab8a182016-10-20 03:18:09 -070068 if (surface instanceof Surface) {
69 eglBase.createSurface((Surface) surface);
70 } else if (surface instanceof SurfaceTexture) {
71 eglBase.createSurface((SurfaceTexture) surface);
72 } else {
73 throw new IllegalStateException("Invalid surface: " + surface);
74 }
magjeddf494b02016-10-07 05:32:35 -070075 eglBase.makeCurrent();
76 // Necessary for YUV frames with odd width.
77 GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1);
78 }
79 }
80 }
81
Xiaolei Yu149533a2017-11-03 07:55:01 +080082 protected final String name;
magjeddf494b02016-10-07 05:32:35 -070083
84 // |renderThreadHandler| is a handler for communicating with |renderThread|, and is synchronized
85 // on |handlerLock|.
86 private final Object handlerLock = new Object();
87 private Handler renderThreadHandler;
88
sakal3a9bc172016-11-30 08:30:05 -080089 private final ArrayList<FrameListenerAndParams> frameListeners = new ArrayList<>();
sakalfb0c5732016-11-03 09:15:34 -070090
magjed9ab8a182016-10-20 03:18:09 -070091 // Variables for fps reduction.
92 private final Object fpsReductionLock = new Object();
93 // Time for when next frame should be rendered.
94 private long nextFrameTimeNs;
95 // Minimum duration between frames when fps reduction is active, or -1 if video is completely
96 // paused.
97 private long minRenderPeriodNs;
98
magjeddf494b02016-10-07 05:32:35 -070099 // EGL and GL resources for drawing YUV/OES textures. After initilization, these are only accessed
100 // from the render thread.
101 private EglBase eglBase;
magjed7cede372017-09-11 06:12:07 -0700102 private final VideoFrameDrawer frameDrawer = new VideoFrameDrawer();
magjeddf494b02016-10-07 05:32:35 -0700103 private RendererCommon.GlDrawer drawer;
magjed7cede372017-09-11 06:12:07 -0700104 private final Matrix drawMatrix = new Matrix();
magjeddf494b02016-10-07 05:32:35 -0700105
106 // Pending frame to render. Serves as a queue with size 1. Synchronized on |frameLock|.
107 private final Object frameLock = new Object();
sakal6bdcefc2017-08-15 01:56:02 -0700108 private VideoFrame pendingFrame;
magjeddf494b02016-10-07 05:32:35 -0700109
110 // These variables are synchronized on |layoutLock|.
111 private final Object layoutLock = new Object();
magjeddf494b02016-10-07 05:32:35 -0700112 private float layoutAspectRatio;
113 // If true, mirrors the video stream horizontally.
114 private boolean mirror;
115
116 // These variables are synchronized on |statisticsLock|.
117 private final Object statisticsLock = new Object();
118 // Total number of video frames received in renderFrame() call.
119 private int framesReceived;
120 // Number of video frames dropped by renderFrame() because previous frame has not been rendered
121 // yet.
122 private int framesDropped;
123 // Number of rendered video frames.
124 private int framesRendered;
magjed9ab8a182016-10-20 03:18:09 -0700125 // Start time for counting these statistics, or 0 if we haven't started measuring yet.
126 private long statisticsStartTimeNs;
magjeddf494b02016-10-07 05:32:35 -0700127 // Time in ns spent in renderFrameOnRenderThread() function.
128 private long renderTimeNs;
magjed9ab8a182016-10-20 03:18:09 -0700129 // Time in ns spent by the render thread in the swapBuffers() function.
130 private long renderSwapBufferTimeNs;
magjeddf494b02016-10-07 05:32:35 -0700131
sakalfb0c5732016-11-03 09:15:34 -0700132 // Used for bitmap capturing.
133 private GlTextureFrameBuffer bitmapTextureFramebuffer;
134
magjed9ab8a182016-10-20 03:18:09 -0700135 private final Runnable logStatisticsRunnable = new Runnable() {
136 @Override
137 public void run() {
138 logStatistics();
139 synchronized (handlerLock) {
140 if (renderThreadHandler != null) {
141 renderThreadHandler.removeCallbacks(logStatisticsRunnable);
142 renderThreadHandler.postDelayed(
143 logStatisticsRunnable, TimeUnit.SECONDS.toMillis(LOG_INTERVAL_SEC));
144 }
145 }
146 }
147 };
148
magjeddf494b02016-10-07 05:32:35 -0700149 private final EglSurfaceCreation eglSurfaceCreationRunnable = new EglSurfaceCreation();
150
151 /**
152 * Standard constructor. The name will be used for the render thread name and included when
153 * logging. In order to render something, you must first call init() and createEglSurface.
154 */
155 public EglRenderer(String name) {
156 this.name = name;
157 }
158
159 /**
160 * Initialize this class, sharing resources with |sharedContext|. The custom |drawer| will be used
161 * for drawing frames on the EGLSurface. This class is responsible for calling release() on
162 * |drawer|. It is allowed to call init() to reinitialize the renderer after a previous
163 * init()/release() cycle.
164 */
165 public void init(final EglBase.Context sharedContext, final int[] configAttributes,
166 RendererCommon.GlDrawer drawer) {
167 synchronized (handlerLock) {
168 if (renderThreadHandler != null) {
169 throw new IllegalStateException(name + "Already initialized");
170 }
171 logD("Initializing EglRenderer");
172 this.drawer = drawer;
173
174 final HandlerThread renderThread = new HandlerThread(name + "EglRenderer");
175 renderThread.start();
176 renderThreadHandler = new Handler(renderThread.getLooper());
177 // Create EGL context on the newly created render thread. It should be possibly to create the
178 // context on this thread and make it current on the render thread, but this causes failure on
179 // some Marvel based JB devices. https://bugs.chromium.org/p/webrtc/issues/detail?id=6350.
sakalbf080602017-08-11 01:42:43 -0700180 ThreadUtils.invokeAtFrontUninterruptibly(renderThreadHandler, () -> {
181 // If sharedContext is null, then texture frames are disabled. This is typically for old
182 // devices that might not be fully spec compliant, so force EGL 1.0 since EGL 1.4 has
183 // caused trouble on some weird devices.
184 if (sharedContext == null) {
185 logD("EglBase10.create context");
186 eglBase = EglBase.createEgl10(configAttributes);
187 } else {
188 logD("EglBase.create shared context");
189 eglBase = EglBase.create(sharedContext, configAttributes);
magjeddf494b02016-10-07 05:32:35 -0700190 }
191 });
magjed9ab8a182016-10-20 03:18:09 -0700192 renderThreadHandler.post(eglSurfaceCreationRunnable);
193 final long currentTimeNs = System.nanoTime();
194 resetStatistics(currentTimeNs);
195 renderThreadHandler.postDelayed(
196 logStatisticsRunnable, TimeUnit.SECONDS.toMillis(LOG_INTERVAL_SEC));
magjeddf494b02016-10-07 05:32:35 -0700197 }
198 }
199
200 public void createEglSurface(Surface surface) {
magjed9ab8a182016-10-20 03:18:09 -0700201 createEglSurfaceInternal(surface);
202 }
203
204 public void createEglSurface(SurfaceTexture surfaceTexture) {
205 createEglSurfaceInternal(surfaceTexture);
206 }
207
208 private void createEglSurfaceInternal(Object surface) {
magjeddf494b02016-10-07 05:32:35 -0700209 eglSurfaceCreationRunnable.setSurface(surface);
magjed9ab8a182016-10-20 03:18:09 -0700210 postToRenderThread(eglSurfaceCreationRunnable);
magjeddf494b02016-10-07 05:32:35 -0700211 }
212
213 /**
214 * Block until any pending frame is returned and all GL resources released, even if an interrupt
215 * occurs. If an interrupt occurs during release(), the interrupt flag will be set. This function
216 * should be called before the Activity is destroyed and the EGLContext is still valid. If you
217 * don't call this function, the GL resources might leak.
218 */
219 public void release() {
magjed9ab8a182016-10-20 03:18:09 -0700220 logD("Releasing.");
magjeddf494b02016-10-07 05:32:35 -0700221 final CountDownLatch eglCleanupBarrier = new CountDownLatch(1);
222 synchronized (handlerLock) {
223 if (renderThreadHandler == null) {
224 logD("Already released");
225 return;
226 }
magjed9ab8a182016-10-20 03:18:09 -0700227 renderThreadHandler.removeCallbacks(logStatisticsRunnable);
magjeddf494b02016-10-07 05:32:35 -0700228 // Release EGL and GL resources on render thread.
sakalbf080602017-08-11 01:42:43 -0700229 renderThreadHandler.postAtFrontOfQueue(() -> {
230 if (drawer != null) {
231 drawer.release();
232 drawer = null;
magjeddf494b02016-10-07 05:32:35 -0700233 }
magjed7cede372017-09-11 06:12:07 -0700234 frameDrawer.release();
sakalbf080602017-08-11 01:42:43 -0700235 if (bitmapTextureFramebuffer != null) {
236 bitmapTextureFramebuffer.release();
237 bitmapTextureFramebuffer = null;
238 }
239 if (eglBase != null) {
240 logD("eglBase detach and release.");
241 eglBase.detachCurrent();
242 eglBase.release();
243 eglBase = null;
244 }
245 eglCleanupBarrier.countDown();
magjeddf494b02016-10-07 05:32:35 -0700246 });
247 final Looper renderLooper = renderThreadHandler.getLooper();
248 // TODO(magjed): Replace this post() with renderLooper.quitSafely() when API support >= 18.
sakalbf080602017-08-11 01:42:43 -0700249 renderThreadHandler.post(() -> {
250 logD("Quitting render thread.");
251 renderLooper.quit();
magjeddf494b02016-10-07 05:32:35 -0700252 });
253 // Don't accept any more frames or messages to the render thread.
254 renderThreadHandler = null;
255 }
256 // Make sure the EGL/GL cleanup posted above is executed.
257 ThreadUtils.awaitUninterruptibly(eglCleanupBarrier);
258 synchronized (frameLock) {
259 if (pendingFrame != null) {
sakal6bdcefc2017-08-15 01:56:02 -0700260 pendingFrame.release();
magjeddf494b02016-10-07 05:32:35 -0700261 pendingFrame = null;
262 }
263 }
magjeddf494b02016-10-07 05:32:35 -0700264 logD("Releasing done.");
265 }
266
267 /**
magjed9ab8a182016-10-20 03:18:09 -0700268 * Reset the statistics logged in logStatistics().
magjeddf494b02016-10-07 05:32:35 -0700269 */
magjed9ab8a182016-10-20 03:18:09 -0700270 private void resetStatistics(long currentTimeNs) {
magjeddf494b02016-10-07 05:32:35 -0700271 synchronized (statisticsLock) {
magjed9ab8a182016-10-20 03:18:09 -0700272 statisticsStartTimeNs = currentTimeNs;
magjeddf494b02016-10-07 05:32:35 -0700273 framesReceived = 0;
274 framesDropped = 0;
275 framesRendered = 0;
magjeddf494b02016-10-07 05:32:35 -0700276 renderTimeNs = 0;
magjed9ab8a182016-10-20 03:18:09 -0700277 renderSwapBufferTimeNs = 0;
278 }
279 }
280
281 public void printStackTrace() {
282 synchronized (handlerLock) {
283 final Thread renderThread =
284 (renderThreadHandler == null) ? null : renderThreadHandler.getLooper().getThread();
285 if (renderThread != null) {
286 final StackTraceElement[] renderStackTrace = renderThread.getStackTrace();
287 if (renderStackTrace.length > 0) {
288 logD("EglRenderer stack trace:");
289 for (StackTraceElement traceElem : renderStackTrace) {
290 logD(traceElem.toString());
291 }
292 }
293 }
magjeddf494b02016-10-07 05:32:35 -0700294 }
295 }
296
297 /**
298 * Set if the video stream should be mirrored or not.
299 */
300 public void setMirror(final boolean mirror) {
301 logD("setMirror: " + mirror);
302 synchronized (layoutLock) {
303 this.mirror = mirror;
304 }
305 }
306
307 /**
308 * Set layout aspect ratio. This is used to crop frames when rendering to avoid stretched video.
309 * Set this to 0 to disable cropping.
310 */
311 public void setLayoutAspectRatio(float layoutAspectRatio) {
312 logD("setLayoutAspectRatio: " + layoutAspectRatio);
313 synchronized (layoutLock) {
314 this.layoutAspectRatio = layoutAspectRatio;
315 }
316 }
317
magjed9ab8a182016-10-20 03:18:09 -0700318 /**
319 * Limit render framerate.
320 *
321 * @param fps Limit render framerate to this value, or use Float.POSITIVE_INFINITY to disable fps
322 * reduction.
323 */
324 public void setFpsReduction(float fps) {
325 logD("setFpsReduction: " + fps);
326 synchronized (fpsReductionLock) {
327 final long previousRenderPeriodNs = minRenderPeriodNs;
328 if (fps <= 0) {
329 minRenderPeriodNs = Long.MAX_VALUE;
330 } else {
331 minRenderPeriodNs = (long) (TimeUnit.SECONDS.toNanos(1) / fps);
332 }
333 if (minRenderPeriodNs != previousRenderPeriodNs) {
334 // Fps reduction changed - reset frame time.
335 nextFrameTimeNs = System.nanoTime();
336 }
337 }
338 }
339
340 public void disableFpsReduction() {
341 setFpsReduction(Float.POSITIVE_INFINITY /* fps */);
342 }
343
344 public void pauseVideo() {
345 setFpsReduction(0 /* fps */);
346 }
347
sakalfb0c5732016-11-03 09:15:34 -0700348 /**
sakal3a9bc172016-11-30 08:30:05 -0800349 * Register a callback to be invoked when a new video frame has been received. This version uses
350 * the drawer of the EglRenderer that was passed in init.
sakalfb0c5732016-11-03 09:15:34 -0700351 *
sakald1516522017-03-13 05:11:48 -0700352 * @param listener The callback to be invoked. The callback will be invoked on the render thread.
353 * It should be lightweight and must not call removeFrameListener.
sakalfb0c5732016-11-03 09:15:34 -0700354 * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is
355 * required.
356 */
sakalbb584352016-11-28 08:53:44 -0800357 public void addFrameListener(final FrameListener listener, final float scale) {
sakal8fdf9572017-05-31 02:43:10 -0700358 addFrameListener(listener, scale, null, false /* applyFpsReduction */);
sakal3a9bc172016-11-30 08:30:05 -0800359 }
360
361 /**
362 * Register a callback to be invoked when a new video frame has been received.
363 *
sakald1516522017-03-13 05:11:48 -0700364 * @param listener The callback to be invoked. The callback will be invoked on the render thread.
365 * It should be lightweight and must not call removeFrameListener.
sakal3a9bc172016-11-30 08:30:05 -0800366 * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is
367 * required.
sakald1516522017-03-13 05:11:48 -0700368 * @param drawer Custom drawer to use for this frame listener or null to use the default one.
sakal3a9bc172016-11-30 08:30:05 -0800369 */
370 public void addFrameListener(
sakald1516522017-03-13 05:11:48 -0700371 final FrameListener listener, final float scale, final RendererCommon.GlDrawer drawerParam) {
sakal8fdf9572017-05-31 02:43:10 -0700372 addFrameListener(listener, scale, drawerParam, false /* applyFpsReduction */);
373 }
374
375 /**
376 * Register a callback to be invoked when a new video frame has been received.
377 *
378 * @param listener The callback to be invoked. The callback will be invoked on the render thread.
379 * It should be lightweight and must not call removeFrameListener.
380 * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is
381 * required.
382 * @param drawer Custom drawer to use for this frame listener or null to use the default one.
383 * @param applyFpsReduction This callback will not be called for frames that have been dropped by
384 * FPS reduction.
385 */
386 public void addFrameListener(final FrameListener listener, final float scale,
387 final RendererCommon.GlDrawer drawerParam, final boolean applyFpsReduction) {
sakalbf080602017-08-11 01:42:43 -0700388 postToRenderThread(() -> {
389 final RendererCommon.GlDrawer listenerDrawer = drawerParam == null ? drawer : drawerParam;
390 frameListeners.add(
391 new FrameListenerAndParams(listener, scale, listenerDrawer, applyFpsReduction));
sakalbb584352016-11-28 08:53:44 -0800392 });
sakalfb0c5732016-11-03 09:15:34 -0700393 }
394
395 /**
396 * Remove any pending callback that was added with addFrameListener. If the callback is not in
sakalbb584352016-11-28 08:53:44 -0800397 * the queue, nothing happens. It is ensured that callback won't be called after this method
398 * returns.
sakalfb0c5732016-11-03 09:15:34 -0700399 *
400 * @param runnable The callback to remove.
401 */
sakalbb584352016-11-28 08:53:44 -0800402 public void removeFrameListener(final FrameListener listener) {
sakald1516522017-03-13 05:11:48 -0700403 if (Thread.currentThread() == renderThreadHandler.getLooper().getThread()) {
404 throw new RuntimeException("removeFrameListener must not be called on the render thread.");
405 }
sakalbb584352016-11-28 08:53:44 -0800406 final CountDownLatch latch = new CountDownLatch(1);
sakalbf080602017-08-11 01:42:43 -0700407 postToRenderThread(() -> {
408 latch.countDown();
409 final Iterator<FrameListenerAndParams> iter = frameListeners.iterator();
410 while (iter.hasNext()) {
411 if (iter.next().listener == listener) {
412 iter.remove();
sakalfb0c5732016-11-03 09:15:34 -0700413 }
414 }
sakalbb584352016-11-28 08:53:44 -0800415 });
416 ThreadUtils.awaitUninterruptibly(latch);
sakalfb0c5732016-11-03 09:15:34 -0700417 }
418
magjeddf494b02016-10-07 05:32:35 -0700419 // VideoRenderer.Callbacks interface.
420 @Override
421 public void renderFrame(VideoRenderer.I420Frame frame) {
sakal6bdcefc2017-08-15 01:56:02 -0700422 VideoFrame videoFrame = frame.toVideoFrame();
423 onFrame(videoFrame);
424 videoFrame.release();
425 }
426
427 // VideoSink interface.
428 @Override
429 public void onFrame(VideoFrame frame) {
magjeddf494b02016-10-07 05:32:35 -0700430 synchronized (statisticsLock) {
431 ++framesReceived;
432 }
magjed9ab8a182016-10-20 03:18:09 -0700433 final boolean dropOldFrame;
magjeddf494b02016-10-07 05:32:35 -0700434 synchronized (handlerLock) {
435 if (renderThreadHandler == null) {
436 logD("Dropping frame - Not initialized or already released.");
magjeddf494b02016-10-07 05:32:35 -0700437 return;
438 }
magjed9ab8a182016-10-20 03:18:09 -0700439 synchronized (frameLock) {
440 dropOldFrame = (pendingFrame != null);
441 if (dropOldFrame) {
sakal6bdcefc2017-08-15 01:56:02 -0700442 pendingFrame.release();
magjeddf494b02016-10-07 05:32:35 -0700443 }
444 pendingFrame = frame;
sakal6bdcefc2017-08-15 01:56:02 -0700445 pendingFrame.retain();
sakalbf080602017-08-11 01:42:43 -0700446 renderThreadHandler.post(this ::renderFrameOnRenderThread);
magjeddf494b02016-10-07 05:32:35 -0700447 }
448 }
magjed9ab8a182016-10-20 03:18:09 -0700449 if (dropOldFrame) {
450 synchronized (statisticsLock) {
451 ++framesDropped;
452 }
453 }
magjeddf494b02016-10-07 05:32:35 -0700454 }
455
456 /**
457 * Release EGL surface. This function will block until the EGL surface is released.
458 */
sakal28ec6bd2016-11-09 01:47:12 -0800459 public void releaseEglSurface(final Runnable completionCallback) {
magjeddf494b02016-10-07 05:32:35 -0700460 // Ensure that the render thread is no longer touching the Surface before returning from this
461 // function.
462 eglSurfaceCreationRunnable.setSurface(null /* surface */);
463 synchronized (handlerLock) {
464 if (renderThreadHandler != null) {
465 renderThreadHandler.removeCallbacks(eglSurfaceCreationRunnable);
sakalbf080602017-08-11 01:42:43 -0700466 renderThreadHandler.postAtFrontOfQueue(() -> {
467 if (eglBase != null) {
468 eglBase.detachCurrent();
469 eglBase.releaseSurface();
magjeddf494b02016-10-07 05:32:35 -0700470 }
sakalbf080602017-08-11 01:42:43 -0700471 completionCallback.run();
magjeddf494b02016-10-07 05:32:35 -0700472 });
sakal28ec6bd2016-11-09 01:47:12 -0800473 return;
magjeddf494b02016-10-07 05:32:35 -0700474 }
475 }
sakal28ec6bd2016-11-09 01:47:12 -0800476 completionCallback.run();
magjeddf494b02016-10-07 05:32:35 -0700477 }
478
479 /**
magjeddf494b02016-10-07 05:32:35 -0700480 * Private helper function to post tasks safely.
481 */
magjed9ab8a182016-10-20 03:18:09 -0700482 private void postToRenderThread(Runnable runnable) {
magjeddf494b02016-10-07 05:32:35 -0700483 synchronized (handlerLock) {
484 if (renderThreadHandler != null) {
485 renderThreadHandler.post(runnable);
486 }
487 }
488 }
489
sakalf25a2202017-05-04 06:06:56 -0700490 private void clearSurfaceOnRenderThread(float r, float g, float b, float a) {
magjeddf494b02016-10-07 05:32:35 -0700491 if (eglBase != null && eglBase.hasSurface()) {
492 logD("clearSurface");
sakalf25a2202017-05-04 06:06:56 -0700493 GLES20.glClearColor(r, g, b, a);
magjeddf494b02016-10-07 05:32:35 -0700494 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
495 eglBase.swapBuffers();
496 }
497 }
498
499 /**
sakalf25a2202017-05-04 06:06:56 -0700500 * Post a task to clear the surface to a transparent uniform color.
magjed9ab8a182016-10-20 03:18:09 -0700501 */
502 public void clearImage() {
sakalf25a2202017-05-04 06:06:56 -0700503 clearImage(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */);
504 }
505
506 /**
507 * Post a task to clear the surface to a specific color.
508 */
509 public void clearImage(final float r, final float g, final float b, final float a) {
magjed9ab8a182016-10-20 03:18:09 -0700510 synchronized (handlerLock) {
511 if (renderThreadHandler == null) {
512 return;
513 }
sakalbf080602017-08-11 01:42:43 -0700514 renderThreadHandler.postAtFrontOfQueue(() -> clearSurfaceOnRenderThread(r, g, b, a));
magjed9ab8a182016-10-20 03:18:09 -0700515 }
516 }
517
518 /**
magjeddf494b02016-10-07 05:32:35 -0700519 * Renders and releases |pendingFrame|.
520 */
521 private void renderFrameOnRenderThread() {
522 // Fetch and render |pendingFrame|.
sakal6bdcefc2017-08-15 01:56:02 -0700523 final VideoFrame frame;
magjeddf494b02016-10-07 05:32:35 -0700524 synchronized (frameLock) {
525 if (pendingFrame == null) {
526 return;
527 }
528 frame = pendingFrame;
529 pendingFrame = null;
530 }
531 if (eglBase == null || !eglBase.hasSurface()) {
532 logD("Dropping frame - No surface");
sakal6bdcefc2017-08-15 01:56:02 -0700533 frame.release();
magjeddf494b02016-10-07 05:32:35 -0700534 return;
535 }
sakald1516522017-03-13 05:11:48 -0700536 // Check if fps reduction is active.
537 final boolean shouldRenderFrame;
538 synchronized (fpsReductionLock) {
539 if (minRenderPeriodNs == Long.MAX_VALUE) {
540 // Rendering is paused.
541 shouldRenderFrame = false;
542 } else if (minRenderPeriodNs <= 0) {
543 // FPS reduction is disabled.
544 shouldRenderFrame = true;
545 } else {
546 final long currentTimeNs = System.nanoTime();
547 if (currentTimeNs < nextFrameTimeNs) {
548 logD("Skipping frame rendering - fps reduction is active.");
549 shouldRenderFrame = false;
550 } else {
551 nextFrameTimeNs += minRenderPeriodNs;
552 // The time for the next frame should always be in the future.
553 nextFrameTimeNs = Math.max(nextFrameTimeNs, currentTimeNs);
554 shouldRenderFrame = true;
555 }
556 }
557 }
magjeddf494b02016-10-07 05:32:35 -0700558
559 final long startTimeNs = System.nanoTime();
magjeddf494b02016-10-07 05:32:35 -0700560
sakal6bdcefc2017-08-15 01:56:02 -0700561 final float frameAspectRatio = frame.getRotatedWidth() / (float) frame.getRotatedHeight();
562 final float drawnAspectRatio;
magjeddf494b02016-10-07 05:32:35 -0700563 synchronized (layoutLock) {
sakal6bdcefc2017-08-15 01:56:02 -0700564 drawnAspectRatio = layoutAspectRatio != 0f ? layoutAspectRatio : frameAspectRatio;
magjeddf494b02016-10-07 05:32:35 -0700565 }
566
sakal6bdcefc2017-08-15 01:56:02 -0700567 final float scaleX;
568 final float scaleY;
569
570 if (frameAspectRatio > drawnAspectRatio) {
571 scaleX = drawnAspectRatio / frameAspectRatio;
572 scaleY = 1f;
573 } else {
574 scaleX = 1f;
575 scaleY = frameAspectRatio / drawnAspectRatio;
576 }
577
magjed7cede372017-09-11 06:12:07 -0700578 drawMatrix.reset();
sakal6bdcefc2017-08-15 01:56:02 -0700579 drawMatrix.preTranslate(0.5f, 0.5f);
sakal6bdcefc2017-08-15 01:56:02 -0700580 if (mirror)
581 drawMatrix.preScale(-1f, 1f);
582 drawMatrix.preScale(scaleX, scaleY);
583 drawMatrix.preTranslate(-0.5f, -0.5f);
584
sakald1516522017-03-13 05:11:48 -0700585 if (shouldRenderFrame) {
586 GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */);
587 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
magjed7cede372017-09-11 06:12:07 -0700588 frameDrawer.drawFrame(frame, drawer, drawMatrix, 0 /* viewportX */, 0 /* viewportY */,
589 eglBase.surfaceWidth(), eglBase.surfaceHeight());
sakald1516522017-03-13 05:11:48 -0700590
591 final long swapBuffersStartTimeNs = System.nanoTime();
592 eglBase.swapBuffers();
593
594 final long currentTimeNs = System.nanoTime();
595 synchronized (statisticsLock) {
596 ++framesRendered;
597 renderTimeNs += (currentTimeNs - startTimeNs);
598 renderSwapBufferTimeNs += (currentTimeNs - swapBuffersStartTimeNs);
599 }
magjeddf494b02016-10-07 05:32:35 -0700600 }
601
magjed7cede372017-09-11 06:12:07 -0700602 notifyCallbacks(frame, shouldRenderFrame);
603 frame.release();
sakalfb0c5732016-11-03 09:15:34 -0700604 }
605
magjed7cede372017-09-11 06:12:07 -0700606 private void notifyCallbacks(VideoFrame frame, boolean wasRendered) {
sakalbb584352016-11-28 08:53:44 -0800607 if (frameListeners.isEmpty())
608 return;
sakalfb0c5732016-11-03 09:15:34 -0700609
magjed7cede372017-09-11 06:12:07 -0700610 drawMatrix.reset();
sakal6bdcefc2017-08-15 01:56:02 -0700611 drawMatrix.preTranslate(0.5f, 0.5f);
sakal6bdcefc2017-08-15 01:56:02 -0700612 if (mirror)
613 drawMatrix.preScale(-1f, 1f);
614 drawMatrix.preScale(1f, -1f); // We want the output to be upside down for Bitmap.
615 drawMatrix.preTranslate(-0.5f, -0.5f);
sakalfb0c5732016-11-03 09:15:34 -0700616
sakal8fdf9572017-05-31 02:43:10 -0700617 Iterator<FrameListenerAndParams> it = frameListeners.iterator();
618 while (it.hasNext()) {
619 FrameListenerAndParams listenerAndParams = it.next();
620 if (!wasRendered && listenerAndParams.applyFpsReduction) {
621 continue;
622 }
623 it.remove();
624
sakal6bdcefc2017-08-15 01:56:02 -0700625 final int scaledWidth = (int) (listenerAndParams.scale * frame.getRotatedWidth());
626 final int scaledHeight = (int) (listenerAndParams.scale * frame.getRotatedHeight());
sakalfb0c5732016-11-03 09:15:34 -0700627
628 if (scaledWidth == 0 || scaledHeight == 0) {
sakal3a9bc172016-11-30 08:30:05 -0800629 listenerAndParams.listener.onFrame(null);
sakalfb0c5732016-11-03 09:15:34 -0700630 continue;
631 }
632
633 if (bitmapTextureFramebuffer == null) {
634 bitmapTextureFramebuffer = new GlTextureFrameBuffer(GLES20.GL_RGBA);
635 }
636 bitmapTextureFramebuffer.setSize(scaledWidth, scaledHeight);
637
638 GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, bitmapTextureFramebuffer.getFrameBufferId());
639 GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0,
640 GLES20.GL_TEXTURE_2D, bitmapTextureFramebuffer.getTextureId(), 0);
641
sakal103988d2017-02-17 09:59:01 -0800642 GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */);
643 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
magjed7cede372017-09-11 06:12:07 -0700644 frameDrawer.drawFrame(frame, listenerAndParams.drawer, drawMatrix, 0 /* viewportX */,
645 0 /* viewportY */, scaledWidth, scaledHeight);
sakalfb0c5732016-11-03 09:15:34 -0700646
647 final ByteBuffer bitmapBuffer = ByteBuffer.allocateDirect(scaledWidth * scaledHeight * 4);
648 GLES20.glViewport(0, 0, scaledWidth, scaledHeight);
649 GLES20.glReadPixels(
650 0, 0, scaledWidth, scaledHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, bitmapBuffer);
651
652 GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
653 GlUtil.checkNoGLES2Error("EglRenderer.notifyCallbacks");
654
655 final Bitmap bitmap = Bitmap.createBitmap(scaledWidth, scaledHeight, Bitmap.Config.ARGB_8888);
656 bitmap.copyPixelsFromBuffer(bitmapBuffer);
sakal3a9bc172016-11-30 08:30:05 -0800657 listenerAndParams.listener.onFrame(bitmap);
sakalfb0c5732016-11-03 09:15:34 -0700658 }
magjeddf494b02016-10-07 05:32:35 -0700659 }
660
magjed9ab8a182016-10-20 03:18:09 -0700661 private String averageTimeAsString(long sumTimeNs, int count) {
662 return (count <= 0) ? "NA" : TimeUnit.NANOSECONDS.toMicros(sumTimeNs / count) + " μs";
663 }
664
magjeddf494b02016-10-07 05:32:35 -0700665 private void logStatistics() {
magjed9ab8a182016-10-20 03:18:09 -0700666 final long currentTimeNs = System.nanoTime();
magjeddf494b02016-10-07 05:32:35 -0700667 synchronized (statisticsLock) {
magjed9ab8a182016-10-20 03:18:09 -0700668 final long elapsedTimeNs = currentTimeNs - statisticsStartTimeNs;
669 if (elapsedTimeNs <= 0) {
670 return;
magjeddf494b02016-10-07 05:32:35 -0700671 }
magjed9ab8a182016-10-20 03:18:09 -0700672 final float renderFps = framesRendered * TimeUnit.SECONDS.toNanos(1) / (float) elapsedTimeNs;
673 logD("Duration: " + TimeUnit.NANOSECONDS.toMillis(elapsedTimeNs) + " ms."
674 + " Frames received: " + framesReceived + "."
675 + " Dropped: " + framesDropped + "."
676 + " Rendered: " + framesRendered + "."
sakal037b93a2017-01-16 04:57:32 -0800677 + " Render fps: " + String.format(Locale.US, "%.1f", renderFps) + "."
magjed9ab8a182016-10-20 03:18:09 -0700678 + " Average render time: " + averageTimeAsString(renderTimeNs, framesRendered) + "."
679 + " Average swapBuffer time: "
680 + averageTimeAsString(renderSwapBufferTimeNs, framesRendered) + ".");
681 resetStatistics(currentTimeNs);
magjeddf494b02016-10-07 05:32:35 -0700682 }
683 }
684
685 private void logD(String string) {
686 Logging.d(TAG, name + string);
687 }
688}