blob: bef34f5a266291f534aff58553be69a67325f117 [file] [log] [blame]
magjeddf494b02016-10-07 05:32:35 -07001/*
2 * Copyright 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11package org.webrtc;
12
sakalfb0c5732016-11-03 09:15:34 -070013import android.graphics.Bitmap;
sakal6bdcefc2017-08-15 01:56:02 -070014import android.graphics.Matrix;
magjed9ab8a182016-10-20 03:18:09 -070015import android.graphics.SurfaceTexture;
magjeddf494b02016-10-07 05:32:35 -070016import android.opengl.GLES20;
17import android.os.Handler;
18import android.os.HandlerThread;
19import android.os.Looper;
Sami Kalliomäki0d26c992018-10-19 12:53:21 +020020import android.os.Message;
magjeddf494b02016-10-07 05:32:35 -070021import android.view.Surface;
sakalfb0c5732016-11-03 09:15:34 -070022import java.nio.ByteBuffer;
Sami Kalliomäki1659e972018-06-04 14:07:58 +020023import java.text.DecimalFormat;
sakalfb0c5732016-11-03 09:15:34 -070024import java.util.ArrayList;
25import java.util.Iterator;
magjeddf494b02016-10-07 05:32:35 -070026import java.util.concurrent.CountDownLatch;
27import java.util.concurrent.TimeUnit;
Sami Kalliomäkie7592d82018-03-22 13:32:44 +010028import javax.annotation.Nullable;
magjeddf494b02016-10-07 05:32:35 -070029
30/**
Magnus Jedvert431f14e2018-06-09 19:41:58 +020031 * Implements VideoSink by displaying the video stream on an EGL Surface. This class is intended to
32 * be used as a helper class for rendering on SurfaceViews and TextureViews.
magjeddf494b02016-10-07 05:32:35 -070033 */
Magnus Jedvert431f14e2018-06-09 19:41:58 +020034public class EglRenderer implements VideoSink {
magjeddf494b02016-10-07 05:32:35 -070035 private static final String TAG = "EglRenderer";
magjed9ab8a182016-10-20 03:18:09 -070036 private static final long LOG_INTERVAL_SEC = 4;
magjeddf494b02016-10-07 05:32:35 -070037
sakalfb0c5732016-11-03 09:15:34 -070038 public interface FrameListener { void onFrame(Bitmap frame); }
39
sakal3a9bc172016-11-30 08:30:05 -080040 private static class FrameListenerAndParams {
sakalfb0c5732016-11-03 09:15:34 -070041 public final FrameListener listener;
sakal3a9bc172016-11-30 08:30:05 -080042 public final float scale;
43 public final RendererCommon.GlDrawer drawer;
sakal8fdf9572017-05-31 02:43:10 -070044 public final boolean applyFpsReduction;
sakalfb0c5732016-11-03 09:15:34 -070045
sakal8fdf9572017-05-31 02:43:10 -070046 public FrameListenerAndParams(FrameListener listener, float scale,
47 RendererCommon.GlDrawer drawer, boolean applyFpsReduction) {
sakalfb0c5732016-11-03 09:15:34 -070048 this.listener = listener;
sakal3a9bc172016-11-30 08:30:05 -080049 this.scale = scale;
50 this.drawer = drawer;
sakal8fdf9572017-05-31 02:43:10 -070051 this.applyFpsReduction = applyFpsReduction;
sakalfb0c5732016-11-03 09:15:34 -070052 }
53 }
54
magjeddf494b02016-10-07 05:32:35 -070055 private class EglSurfaceCreation implements Runnable {
magjed9ab8a182016-10-20 03:18:09 -070056 private Object surface;
magjeddf494b02016-10-07 05:32:35 -070057
Mirko Bonadei12251b62017-11-05 19:35:31 -080058 // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression.
59 @SuppressWarnings("NoSynchronizedMethodCheck")
magjed9ab8a182016-10-20 03:18:09 -070060 public synchronized void setSurface(Object surface) {
magjeddf494b02016-10-07 05:32:35 -070061 this.surface = surface;
62 }
63
64 @Override
Mirko Bonadei12251b62017-11-05 19:35:31 -080065 // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression.
66 @SuppressWarnings("NoSynchronizedMethodCheck")
magjeddf494b02016-10-07 05:32:35 -070067 public synchronized void run() {
68 if (surface != null && eglBase != null && !eglBase.hasSurface()) {
magjed9ab8a182016-10-20 03:18:09 -070069 if (surface instanceof Surface) {
70 eglBase.createSurface((Surface) surface);
71 } else if (surface instanceof SurfaceTexture) {
72 eglBase.createSurface((SurfaceTexture) surface);
73 } else {
74 throw new IllegalStateException("Invalid surface: " + surface);
75 }
magjeddf494b02016-10-07 05:32:35 -070076 eglBase.makeCurrent();
77 // Necessary for YUV frames with odd width.
78 GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1);
79 }
80 }
81 }
82
Sami Kalliomäki0d26c992018-10-19 12:53:21 +020083 /**
84 * Handler that triggers a callback when an uncaught exception happens when handling a message.
85 */
86 private static class HandlerWithExceptionCallback extends Handler {
87 private final Runnable exceptionCallback;
88
89 public HandlerWithExceptionCallback(Looper looper, Runnable exceptionCallback) {
90 super(looper);
91 this.exceptionCallback = exceptionCallback;
92 }
93
94 @Override
95 public void dispatchMessage(Message msg) {
96 try {
97 super.dispatchMessage(msg);
98 } catch (Exception e) {
99 Logging.e(TAG, "Exception on EglRenderer thread", e);
100 exceptionCallback.run();
101 throw e;
102 }
103 }
104 }
105
Xiaolei Yu149533a2017-11-03 07:55:01 +0800106 protected final String name;
magjeddf494b02016-10-07 05:32:35 -0700107
108 // |renderThreadHandler| is a handler for communicating with |renderThread|, and is synchronized
109 // on |handlerLock|.
110 private final Object handlerLock = new Object();
Sami Kalliomäkie7592d82018-03-22 13:32:44 +0100111 @Nullable private Handler renderThreadHandler;
magjeddf494b02016-10-07 05:32:35 -0700112
sakal3a9bc172016-11-30 08:30:05 -0800113 private final ArrayList<FrameListenerAndParams> frameListeners = new ArrayList<>();
sakalfb0c5732016-11-03 09:15:34 -0700114
magjed9ab8a182016-10-20 03:18:09 -0700115 // Variables for fps reduction.
116 private final Object fpsReductionLock = new Object();
117 // Time for when next frame should be rendered.
118 private long nextFrameTimeNs;
119 // Minimum duration between frames when fps reduction is active, or -1 if video is completely
120 // paused.
121 private long minRenderPeriodNs;
122
magjeddf494b02016-10-07 05:32:35 -0700123 // EGL and GL resources for drawing YUV/OES textures. After initilization, these are only accessed
124 // from the render thread.
Sami Kalliomäkie7592d82018-03-22 13:32:44 +0100125 @Nullable private EglBase eglBase;
magjed7cede372017-09-11 06:12:07 -0700126 private final VideoFrameDrawer frameDrawer = new VideoFrameDrawer();
Sami Kalliomäkie7592d82018-03-22 13:32:44 +0100127 @Nullable private RendererCommon.GlDrawer drawer;
magjed7cede372017-09-11 06:12:07 -0700128 private final Matrix drawMatrix = new Matrix();
magjeddf494b02016-10-07 05:32:35 -0700129
130 // Pending frame to render. Serves as a queue with size 1. Synchronized on |frameLock|.
131 private final Object frameLock = new Object();
Sami Kalliomäkie7592d82018-03-22 13:32:44 +0100132 @Nullable private VideoFrame pendingFrame;
magjeddf494b02016-10-07 05:32:35 -0700133
134 // These variables are synchronized on |layoutLock|.
135 private final Object layoutLock = new Object();
magjeddf494b02016-10-07 05:32:35 -0700136 private float layoutAspectRatio;
137 // If true, mirrors the video stream horizontally.
138 private boolean mirror;
139
140 // These variables are synchronized on |statisticsLock|.
141 private final Object statisticsLock = new Object();
142 // Total number of video frames received in renderFrame() call.
143 private int framesReceived;
144 // Number of video frames dropped by renderFrame() because previous frame has not been rendered
145 // yet.
146 private int framesDropped;
147 // Number of rendered video frames.
148 private int framesRendered;
magjed9ab8a182016-10-20 03:18:09 -0700149 // Start time for counting these statistics, or 0 if we haven't started measuring yet.
150 private long statisticsStartTimeNs;
magjeddf494b02016-10-07 05:32:35 -0700151 // Time in ns spent in renderFrameOnRenderThread() function.
152 private long renderTimeNs;
magjed9ab8a182016-10-20 03:18:09 -0700153 // Time in ns spent by the render thread in the swapBuffers() function.
154 private long renderSwapBufferTimeNs;
magjeddf494b02016-10-07 05:32:35 -0700155
sakalfb0c5732016-11-03 09:15:34 -0700156 // Used for bitmap capturing.
Magnus Jedvert2ed62b32018-04-11 14:25:14 +0200157 private final GlTextureFrameBuffer bitmapTextureFramebuffer =
158 new GlTextureFrameBuffer(GLES20.GL_RGBA);
sakalfb0c5732016-11-03 09:15:34 -0700159
magjed9ab8a182016-10-20 03:18:09 -0700160 private final Runnable logStatisticsRunnable = new Runnable() {
161 @Override
162 public void run() {
163 logStatistics();
164 synchronized (handlerLock) {
165 if (renderThreadHandler != null) {
166 renderThreadHandler.removeCallbacks(logStatisticsRunnable);
167 renderThreadHandler.postDelayed(
168 logStatisticsRunnable, TimeUnit.SECONDS.toMillis(LOG_INTERVAL_SEC));
169 }
170 }
171 }
172 };
173
magjeddf494b02016-10-07 05:32:35 -0700174 private final EglSurfaceCreation eglSurfaceCreationRunnable = new EglSurfaceCreation();
175
176 /**
177 * Standard constructor. The name will be used for the render thread name and included when
178 * logging. In order to render something, you must first call init() and createEglSurface.
179 */
180 public EglRenderer(String name) {
181 this.name = name;
182 }
183
184 /**
185 * Initialize this class, sharing resources with |sharedContext|. The custom |drawer| will be used
186 * for drawing frames on the EGLSurface. This class is responsible for calling release() on
187 * |drawer|. It is allowed to call init() to reinitialize the renderer after a previous
188 * init()/release() cycle.
189 */
Sami Kalliomäkie7592d82018-03-22 13:32:44 +0100190 public void init(@Nullable final EglBase.Context sharedContext, final int[] configAttributes,
magjeddf494b02016-10-07 05:32:35 -0700191 RendererCommon.GlDrawer drawer) {
192 synchronized (handlerLock) {
193 if (renderThreadHandler != null) {
194 throw new IllegalStateException(name + "Already initialized");
195 }
196 logD("Initializing EglRenderer");
197 this.drawer = drawer;
198
199 final HandlerThread renderThread = new HandlerThread(name + "EglRenderer");
200 renderThread.start();
Sami Kalliomäki0d26c992018-10-19 12:53:21 +0200201 renderThreadHandler =
202 new HandlerWithExceptionCallback(renderThread.getLooper(), new Runnable() {
203 @Override
204 public void run() {
205 synchronized (handlerLock) {
206 renderThreadHandler = null;
207 }
208 }
209 });
magjeddf494b02016-10-07 05:32:35 -0700210 // Create EGL context on the newly created render thread. It should be possibly to create the
211 // context on this thread and make it current on the render thread, but this causes failure on
212 // some Marvel based JB devices. https://bugs.chromium.org/p/webrtc/issues/detail?id=6350.
sakalbf080602017-08-11 01:42:43 -0700213 ThreadUtils.invokeAtFrontUninterruptibly(renderThreadHandler, () -> {
214 // If sharedContext is null, then texture frames are disabled. This is typically for old
215 // devices that might not be fully spec compliant, so force EGL 1.0 since EGL 1.4 has
216 // caused trouble on some weird devices.
217 if (sharedContext == null) {
218 logD("EglBase10.create context");
219 eglBase = EglBase.createEgl10(configAttributes);
220 } else {
221 logD("EglBase.create shared context");
222 eglBase = EglBase.create(sharedContext, configAttributes);
magjeddf494b02016-10-07 05:32:35 -0700223 }
224 });
magjed9ab8a182016-10-20 03:18:09 -0700225 renderThreadHandler.post(eglSurfaceCreationRunnable);
226 final long currentTimeNs = System.nanoTime();
227 resetStatistics(currentTimeNs);
228 renderThreadHandler.postDelayed(
229 logStatisticsRunnable, TimeUnit.SECONDS.toMillis(LOG_INTERVAL_SEC));
magjeddf494b02016-10-07 05:32:35 -0700230 }
231 }
232
233 public void createEglSurface(Surface surface) {
magjed9ab8a182016-10-20 03:18:09 -0700234 createEglSurfaceInternal(surface);
235 }
236
237 public void createEglSurface(SurfaceTexture surfaceTexture) {
238 createEglSurfaceInternal(surfaceTexture);
239 }
240
241 private void createEglSurfaceInternal(Object surface) {
magjeddf494b02016-10-07 05:32:35 -0700242 eglSurfaceCreationRunnable.setSurface(surface);
magjed9ab8a182016-10-20 03:18:09 -0700243 postToRenderThread(eglSurfaceCreationRunnable);
magjeddf494b02016-10-07 05:32:35 -0700244 }
245
246 /**
247 * Block until any pending frame is returned and all GL resources released, even if an interrupt
248 * occurs. If an interrupt occurs during release(), the interrupt flag will be set. This function
249 * should be called before the Activity is destroyed and the EGLContext is still valid. If you
250 * don't call this function, the GL resources might leak.
251 */
252 public void release() {
magjed9ab8a182016-10-20 03:18:09 -0700253 logD("Releasing.");
magjeddf494b02016-10-07 05:32:35 -0700254 final CountDownLatch eglCleanupBarrier = new CountDownLatch(1);
255 synchronized (handlerLock) {
256 if (renderThreadHandler == null) {
257 logD("Already released");
258 return;
259 }
magjed9ab8a182016-10-20 03:18:09 -0700260 renderThreadHandler.removeCallbacks(logStatisticsRunnable);
magjeddf494b02016-10-07 05:32:35 -0700261 // Release EGL and GL resources on render thread.
sakalbf080602017-08-11 01:42:43 -0700262 renderThreadHandler.postAtFrontOfQueue(() -> {
263 if (drawer != null) {
264 drawer.release();
265 drawer = null;
magjeddf494b02016-10-07 05:32:35 -0700266 }
magjed7cede372017-09-11 06:12:07 -0700267 frameDrawer.release();
Magnus Jedvert2ed62b32018-04-11 14:25:14 +0200268 bitmapTextureFramebuffer.release();
sakalbf080602017-08-11 01:42:43 -0700269 if (eglBase != null) {
270 logD("eglBase detach and release.");
271 eglBase.detachCurrent();
272 eglBase.release();
273 eglBase = null;
274 }
Sami Kalliomäki8ebac242017-11-08 17:13:13 +0100275 frameListeners.clear();
sakalbf080602017-08-11 01:42:43 -0700276 eglCleanupBarrier.countDown();
magjeddf494b02016-10-07 05:32:35 -0700277 });
278 final Looper renderLooper = renderThreadHandler.getLooper();
279 // TODO(magjed): Replace this post() with renderLooper.quitSafely() when API support >= 18.
sakalbf080602017-08-11 01:42:43 -0700280 renderThreadHandler.post(() -> {
281 logD("Quitting render thread.");
282 renderLooper.quit();
magjeddf494b02016-10-07 05:32:35 -0700283 });
284 // Don't accept any more frames or messages to the render thread.
285 renderThreadHandler = null;
286 }
287 // Make sure the EGL/GL cleanup posted above is executed.
288 ThreadUtils.awaitUninterruptibly(eglCleanupBarrier);
289 synchronized (frameLock) {
290 if (pendingFrame != null) {
sakal6bdcefc2017-08-15 01:56:02 -0700291 pendingFrame.release();
magjeddf494b02016-10-07 05:32:35 -0700292 pendingFrame = null;
293 }
294 }
magjeddf494b02016-10-07 05:32:35 -0700295 logD("Releasing done.");
296 }
297
298 /**
magjed9ab8a182016-10-20 03:18:09 -0700299 * Reset the statistics logged in logStatistics().
magjeddf494b02016-10-07 05:32:35 -0700300 */
magjed9ab8a182016-10-20 03:18:09 -0700301 private void resetStatistics(long currentTimeNs) {
magjeddf494b02016-10-07 05:32:35 -0700302 synchronized (statisticsLock) {
magjed9ab8a182016-10-20 03:18:09 -0700303 statisticsStartTimeNs = currentTimeNs;
magjeddf494b02016-10-07 05:32:35 -0700304 framesReceived = 0;
305 framesDropped = 0;
306 framesRendered = 0;
magjeddf494b02016-10-07 05:32:35 -0700307 renderTimeNs = 0;
magjed9ab8a182016-10-20 03:18:09 -0700308 renderSwapBufferTimeNs = 0;
309 }
310 }
311
312 public void printStackTrace() {
313 synchronized (handlerLock) {
314 final Thread renderThread =
315 (renderThreadHandler == null) ? null : renderThreadHandler.getLooper().getThread();
316 if (renderThread != null) {
317 final StackTraceElement[] renderStackTrace = renderThread.getStackTrace();
318 if (renderStackTrace.length > 0) {
319 logD("EglRenderer stack trace:");
320 for (StackTraceElement traceElem : renderStackTrace) {
321 logD(traceElem.toString());
322 }
323 }
324 }
magjeddf494b02016-10-07 05:32:35 -0700325 }
326 }
327
328 /**
329 * Set if the video stream should be mirrored or not.
330 */
331 public void setMirror(final boolean mirror) {
332 logD("setMirror: " + mirror);
333 synchronized (layoutLock) {
334 this.mirror = mirror;
335 }
336 }
337
338 /**
339 * Set layout aspect ratio. This is used to crop frames when rendering to avoid stretched video.
340 * Set this to 0 to disable cropping.
341 */
342 public void setLayoutAspectRatio(float layoutAspectRatio) {
343 logD("setLayoutAspectRatio: " + layoutAspectRatio);
344 synchronized (layoutLock) {
345 this.layoutAspectRatio = layoutAspectRatio;
346 }
347 }
348
magjed9ab8a182016-10-20 03:18:09 -0700349 /**
350 * Limit render framerate.
351 *
352 * @param fps Limit render framerate to this value, or use Float.POSITIVE_INFINITY to disable fps
353 * reduction.
354 */
355 public void setFpsReduction(float fps) {
356 logD("setFpsReduction: " + fps);
357 synchronized (fpsReductionLock) {
358 final long previousRenderPeriodNs = minRenderPeriodNs;
359 if (fps <= 0) {
360 minRenderPeriodNs = Long.MAX_VALUE;
361 } else {
362 minRenderPeriodNs = (long) (TimeUnit.SECONDS.toNanos(1) / fps);
363 }
364 if (minRenderPeriodNs != previousRenderPeriodNs) {
365 // Fps reduction changed - reset frame time.
366 nextFrameTimeNs = System.nanoTime();
367 }
368 }
369 }
370
371 public void disableFpsReduction() {
372 setFpsReduction(Float.POSITIVE_INFINITY /* fps */);
373 }
374
375 public void pauseVideo() {
376 setFpsReduction(0 /* fps */);
377 }
378
sakalfb0c5732016-11-03 09:15:34 -0700379 /**
sakal3a9bc172016-11-30 08:30:05 -0800380 * Register a callback to be invoked when a new video frame has been received. This version uses
381 * the drawer of the EglRenderer that was passed in init.
sakalfb0c5732016-11-03 09:15:34 -0700382 *
sakald1516522017-03-13 05:11:48 -0700383 * @param listener The callback to be invoked. The callback will be invoked on the render thread.
384 * It should be lightweight and must not call removeFrameListener.
sakalfb0c5732016-11-03 09:15:34 -0700385 * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is
386 * required.
387 */
sakalbb584352016-11-28 08:53:44 -0800388 public void addFrameListener(final FrameListener listener, final float scale) {
sakal8fdf9572017-05-31 02:43:10 -0700389 addFrameListener(listener, scale, null, false /* applyFpsReduction */);
sakal3a9bc172016-11-30 08:30:05 -0800390 }
391
392 /**
393 * Register a callback to be invoked when a new video frame has been received.
394 *
sakald1516522017-03-13 05:11:48 -0700395 * @param listener The callback to be invoked. The callback will be invoked on the render thread.
396 * It should be lightweight and must not call removeFrameListener.
sakal3a9bc172016-11-30 08:30:05 -0800397 * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is
398 * required.
sakald1516522017-03-13 05:11:48 -0700399 * @param drawer Custom drawer to use for this frame listener or null to use the default one.
sakal3a9bc172016-11-30 08:30:05 -0800400 */
401 public void addFrameListener(
sakald1516522017-03-13 05:11:48 -0700402 final FrameListener listener, final float scale, final RendererCommon.GlDrawer drawerParam) {
sakal8fdf9572017-05-31 02:43:10 -0700403 addFrameListener(listener, scale, drawerParam, false /* applyFpsReduction */);
404 }
405
406 /**
407 * Register a callback to be invoked when a new video frame has been received.
408 *
409 * @param listener The callback to be invoked. The callback will be invoked on the render thread.
410 * It should be lightweight and must not call removeFrameListener.
411 * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is
412 * required.
413 * @param drawer Custom drawer to use for this frame listener or null to use the default one.
414 * @param applyFpsReduction This callback will not be called for frames that have been dropped by
415 * FPS reduction.
416 */
417 public void addFrameListener(final FrameListener listener, final float scale,
Sami Kalliomäkie7592d82018-03-22 13:32:44 +0100418 @Nullable final RendererCommon.GlDrawer drawerParam, final boolean applyFpsReduction) {
sakalbf080602017-08-11 01:42:43 -0700419 postToRenderThread(() -> {
420 final RendererCommon.GlDrawer listenerDrawer = drawerParam == null ? drawer : drawerParam;
421 frameListeners.add(
422 new FrameListenerAndParams(listener, scale, listenerDrawer, applyFpsReduction));
sakalbb584352016-11-28 08:53:44 -0800423 });
sakalfb0c5732016-11-03 09:15:34 -0700424 }
425
426 /**
427 * Remove any pending callback that was added with addFrameListener. If the callback is not in
sakalbb584352016-11-28 08:53:44 -0800428 * the queue, nothing happens. It is ensured that callback won't be called after this method
429 * returns.
sakalfb0c5732016-11-03 09:15:34 -0700430 *
431 * @param runnable The callback to remove.
432 */
sakalbb584352016-11-28 08:53:44 -0800433 public void removeFrameListener(final FrameListener listener) {
434 final CountDownLatch latch = new CountDownLatch(1);
Sami Kalliomäki8ebac242017-11-08 17:13:13 +0100435 synchronized (handlerLock) {
436 if (renderThreadHandler == null) {
437 return;
sakalfb0c5732016-11-03 09:15:34 -0700438 }
Sami Kalliomäki8ebac242017-11-08 17:13:13 +0100439 if (Thread.currentThread() == renderThreadHandler.getLooper().getThread()) {
440 throw new RuntimeException("removeFrameListener must not be called on the render thread.");
441 }
442 postToRenderThread(() -> {
443 latch.countDown();
444 final Iterator<FrameListenerAndParams> iter = frameListeners.iterator();
445 while (iter.hasNext()) {
446 if (iter.next().listener == listener) {
447 iter.remove();
448 }
449 }
450 });
451 }
sakalbb584352016-11-28 08:53:44 -0800452 ThreadUtils.awaitUninterruptibly(latch);
sakalfb0c5732016-11-03 09:15:34 -0700453 }
454
sakal6bdcefc2017-08-15 01:56:02 -0700455 // VideoSink interface.
456 @Override
457 public void onFrame(VideoFrame frame) {
magjeddf494b02016-10-07 05:32:35 -0700458 synchronized (statisticsLock) {
459 ++framesReceived;
460 }
magjed9ab8a182016-10-20 03:18:09 -0700461 final boolean dropOldFrame;
magjeddf494b02016-10-07 05:32:35 -0700462 synchronized (handlerLock) {
463 if (renderThreadHandler == null) {
464 logD("Dropping frame - Not initialized or already released.");
magjeddf494b02016-10-07 05:32:35 -0700465 return;
466 }
magjed9ab8a182016-10-20 03:18:09 -0700467 synchronized (frameLock) {
468 dropOldFrame = (pendingFrame != null);
469 if (dropOldFrame) {
sakal6bdcefc2017-08-15 01:56:02 -0700470 pendingFrame.release();
magjeddf494b02016-10-07 05:32:35 -0700471 }
472 pendingFrame = frame;
sakal6bdcefc2017-08-15 01:56:02 -0700473 pendingFrame.retain();
sakalbf080602017-08-11 01:42:43 -0700474 renderThreadHandler.post(this ::renderFrameOnRenderThread);
magjeddf494b02016-10-07 05:32:35 -0700475 }
476 }
magjed9ab8a182016-10-20 03:18:09 -0700477 if (dropOldFrame) {
478 synchronized (statisticsLock) {
479 ++framesDropped;
480 }
481 }
magjeddf494b02016-10-07 05:32:35 -0700482 }
483
484 /**
485 * Release EGL surface. This function will block until the EGL surface is released.
486 */
sakal28ec6bd2016-11-09 01:47:12 -0800487 public void releaseEglSurface(final Runnable completionCallback) {
magjeddf494b02016-10-07 05:32:35 -0700488 // Ensure that the render thread is no longer touching the Surface before returning from this
489 // function.
490 eglSurfaceCreationRunnable.setSurface(null /* surface */);
491 synchronized (handlerLock) {
492 if (renderThreadHandler != null) {
493 renderThreadHandler.removeCallbacks(eglSurfaceCreationRunnable);
sakalbf080602017-08-11 01:42:43 -0700494 renderThreadHandler.postAtFrontOfQueue(() -> {
495 if (eglBase != null) {
496 eglBase.detachCurrent();
497 eglBase.releaseSurface();
magjeddf494b02016-10-07 05:32:35 -0700498 }
sakalbf080602017-08-11 01:42:43 -0700499 completionCallback.run();
magjeddf494b02016-10-07 05:32:35 -0700500 });
sakal28ec6bd2016-11-09 01:47:12 -0800501 return;
magjeddf494b02016-10-07 05:32:35 -0700502 }
503 }
sakal28ec6bd2016-11-09 01:47:12 -0800504 completionCallback.run();
magjeddf494b02016-10-07 05:32:35 -0700505 }
506
507 /**
magjeddf494b02016-10-07 05:32:35 -0700508 * Private helper function to post tasks safely.
509 */
magjed9ab8a182016-10-20 03:18:09 -0700510 private void postToRenderThread(Runnable runnable) {
magjeddf494b02016-10-07 05:32:35 -0700511 synchronized (handlerLock) {
512 if (renderThreadHandler != null) {
513 renderThreadHandler.post(runnable);
514 }
515 }
516 }
517
sakalf25a2202017-05-04 06:06:56 -0700518 private void clearSurfaceOnRenderThread(float r, float g, float b, float a) {
magjeddf494b02016-10-07 05:32:35 -0700519 if (eglBase != null && eglBase.hasSurface()) {
520 logD("clearSurface");
sakalf25a2202017-05-04 06:06:56 -0700521 GLES20.glClearColor(r, g, b, a);
magjeddf494b02016-10-07 05:32:35 -0700522 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
523 eglBase.swapBuffers();
524 }
525 }
526
527 /**
sakalf25a2202017-05-04 06:06:56 -0700528 * Post a task to clear the surface to a transparent uniform color.
magjed9ab8a182016-10-20 03:18:09 -0700529 */
530 public void clearImage() {
sakalf25a2202017-05-04 06:06:56 -0700531 clearImage(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */);
532 }
533
534 /**
535 * Post a task to clear the surface to a specific color.
536 */
537 public void clearImage(final float r, final float g, final float b, final float a) {
magjed9ab8a182016-10-20 03:18:09 -0700538 synchronized (handlerLock) {
539 if (renderThreadHandler == null) {
540 return;
541 }
sakalbf080602017-08-11 01:42:43 -0700542 renderThreadHandler.postAtFrontOfQueue(() -> clearSurfaceOnRenderThread(r, g, b, a));
magjed9ab8a182016-10-20 03:18:09 -0700543 }
544 }
545
546 /**
magjeddf494b02016-10-07 05:32:35 -0700547 * Renders and releases |pendingFrame|.
548 */
549 private void renderFrameOnRenderThread() {
550 // Fetch and render |pendingFrame|.
sakal6bdcefc2017-08-15 01:56:02 -0700551 final VideoFrame frame;
magjeddf494b02016-10-07 05:32:35 -0700552 synchronized (frameLock) {
553 if (pendingFrame == null) {
554 return;
555 }
556 frame = pendingFrame;
557 pendingFrame = null;
558 }
559 if (eglBase == null || !eglBase.hasSurface()) {
560 logD("Dropping frame - No surface");
sakal6bdcefc2017-08-15 01:56:02 -0700561 frame.release();
magjeddf494b02016-10-07 05:32:35 -0700562 return;
563 }
sakald1516522017-03-13 05:11:48 -0700564 // Check if fps reduction is active.
565 final boolean shouldRenderFrame;
566 synchronized (fpsReductionLock) {
567 if (minRenderPeriodNs == Long.MAX_VALUE) {
568 // Rendering is paused.
569 shouldRenderFrame = false;
570 } else if (minRenderPeriodNs <= 0) {
571 // FPS reduction is disabled.
572 shouldRenderFrame = true;
573 } else {
574 final long currentTimeNs = System.nanoTime();
575 if (currentTimeNs < nextFrameTimeNs) {
576 logD("Skipping frame rendering - fps reduction is active.");
577 shouldRenderFrame = false;
578 } else {
579 nextFrameTimeNs += minRenderPeriodNs;
580 // The time for the next frame should always be in the future.
581 nextFrameTimeNs = Math.max(nextFrameTimeNs, currentTimeNs);
582 shouldRenderFrame = true;
583 }
584 }
585 }
magjeddf494b02016-10-07 05:32:35 -0700586
587 final long startTimeNs = System.nanoTime();
magjeddf494b02016-10-07 05:32:35 -0700588
sakal6bdcefc2017-08-15 01:56:02 -0700589 final float frameAspectRatio = frame.getRotatedWidth() / (float) frame.getRotatedHeight();
590 final float drawnAspectRatio;
magjeddf494b02016-10-07 05:32:35 -0700591 synchronized (layoutLock) {
sakal6bdcefc2017-08-15 01:56:02 -0700592 drawnAspectRatio = layoutAspectRatio != 0f ? layoutAspectRatio : frameAspectRatio;
magjeddf494b02016-10-07 05:32:35 -0700593 }
594
sakal6bdcefc2017-08-15 01:56:02 -0700595 final float scaleX;
596 final float scaleY;
597
598 if (frameAspectRatio > drawnAspectRatio) {
599 scaleX = drawnAspectRatio / frameAspectRatio;
600 scaleY = 1f;
601 } else {
602 scaleX = 1f;
603 scaleY = frameAspectRatio / drawnAspectRatio;
604 }
605
magjed7cede372017-09-11 06:12:07 -0700606 drawMatrix.reset();
sakal6bdcefc2017-08-15 01:56:02 -0700607 drawMatrix.preTranslate(0.5f, 0.5f);
sakal6bdcefc2017-08-15 01:56:02 -0700608 if (mirror)
609 drawMatrix.preScale(-1f, 1f);
610 drawMatrix.preScale(scaleX, scaleY);
611 drawMatrix.preTranslate(-0.5f, -0.5f);
612
sakald1516522017-03-13 05:11:48 -0700613 if (shouldRenderFrame) {
614 GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */);
615 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
magjed7cede372017-09-11 06:12:07 -0700616 frameDrawer.drawFrame(frame, drawer, drawMatrix, 0 /* viewportX */, 0 /* viewportY */,
617 eglBase.surfaceWidth(), eglBase.surfaceHeight());
sakald1516522017-03-13 05:11:48 -0700618
619 final long swapBuffersStartTimeNs = System.nanoTime();
620 eglBase.swapBuffers();
621
622 final long currentTimeNs = System.nanoTime();
623 synchronized (statisticsLock) {
624 ++framesRendered;
625 renderTimeNs += (currentTimeNs - startTimeNs);
626 renderSwapBufferTimeNs += (currentTimeNs - swapBuffersStartTimeNs);
627 }
magjeddf494b02016-10-07 05:32:35 -0700628 }
629
magjed7cede372017-09-11 06:12:07 -0700630 notifyCallbacks(frame, shouldRenderFrame);
631 frame.release();
sakalfb0c5732016-11-03 09:15:34 -0700632 }
633
magjed7cede372017-09-11 06:12:07 -0700634 private void notifyCallbacks(VideoFrame frame, boolean wasRendered) {
sakalbb584352016-11-28 08:53:44 -0800635 if (frameListeners.isEmpty())
636 return;
sakalfb0c5732016-11-03 09:15:34 -0700637
magjed7cede372017-09-11 06:12:07 -0700638 drawMatrix.reset();
sakal6bdcefc2017-08-15 01:56:02 -0700639 drawMatrix.preTranslate(0.5f, 0.5f);
sakal6bdcefc2017-08-15 01:56:02 -0700640 if (mirror)
641 drawMatrix.preScale(-1f, 1f);
642 drawMatrix.preScale(1f, -1f); // We want the output to be upside down for Bitmap.
643 drawMatrix.preTranslate(-0.5f, -0.5f);
sakalfb0c5732016-11-03 09:15:34 -0700644
sakal8fdf9572017-05-31 02:43:10 -0700645 Iterator<FrameListenerAndParams> it = frameListeners.iterator();
646 while (it.hasNext()) {
647 FrameListenerAndParams listenerAndParams = it.next();
648 if (!wasRendered && listenerAndParams.applyFpsReduction) {
649 continue;
650 }
651 it.remove();
652
sakal6bdcefc2017-08-15 01:56:02 -0700653 final int scaledWidth = (int) (listenerAndParams.scale * frame.getRotatedWidth());
654 final int scaledHeight = (int) (listenerAndParams.scale * frame.getRotatedHeight());
sakalfb0c5732016-11-03 09:15:34 -0700655
656 if (scaledWidth == 0 || scaledHeight == 0) {
sakal3a9bc172016-11-30 08:30:05 -0800657 listenerAndParams.listener.onFrame(null);
sakalfb0c5732016-11-03 09:15:34 -0700658 continue;
659 }
660
sakalfb0c5732016-11-03 09:15:34 -0700661 bitmapTextureFramebuffer.setSize(scaledWidth, scaledHeight);
662
663 GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, bitmapTextureFramebuffer.getFrameBufferId());
664 GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0,
665 GLES20.GL_TEXTURE_2D, bitmapTextureFramebuffer.getTextureId(), 0);
666
sakal103988d2017-02-17 09:59:01 -0800667 GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */);
668 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
magjed7cede372017-09-11 06:12:07 -0700669 frameDrawer.drawFrame(frame, listenerAndParams.drawer, drawMatrix, 0 /* viewportX */,
670 0 /* viewportY */, scaledWidth, scaledHeight);
sakalfb0c5732016-11-03 09:15:34 -0700671
672 final ByteBuffer bitmapBuffer = ByteBuffer.allocateDirect(scaledWidth * scaledHeight * 4);
673 GLES20.glViewport(0, 0, scaledWidth, scaledHeight);
674 GLES20.glReadPixels(
675 0, 0, scaledWidth, scaledHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, bitmapBuffer);
676
677 GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
678 GlUtil.checkNoGLES2Error("EglRenderer.notifyCallbacks");
679
680 final Bitmap bitmap = Bitmap.createBitmap(scaledWidth, scaledHeight, Bitmap.Config.ARGB_8888);
681 bitmap.copyPixelsFromBuffer(bitmapBuffer);
sakal3a9bc172016-11-30 08:30:05 -0800682 listenerAndParams.listener.onFrame(bitmap);
sakalfb0c5732016-11-03 09:15:34 -0700683 }
magjeddf494b02016-10-07 05:32:35 -0700684 }
685
magjed9ab8a182016-10-20 03:18:09 -0700686 private String averageTimeAsString(long sumTimeNs, int count) {
687 return (count <= 0) ? "NA" : TimeUnit.NANOSECONDS.toMicros(sumTimeNs / count) + " μs";
688 }
689
magjeddf494b02016-10-07 05:32:35 -0700690 private void logStatistics() {
Sami Kalliomäki1659e972018-06-04 14:07:58 +0200691 final DecimalFormat fpsFormat = new DecimalFormat("#.0");
magjed9ab8a182016-10-20 03:18:09 -0700692 final long currentTimeNs = System.nanoTime();
magjeddf494b02016-10-07 05:32:35 -0700693 synchronized (statisticsLock) {
magjed9ab8a182016-10-20 03:18:09 -0700694 final long elapsedTimeNs = currentTimeNs - statisticsStartTimeNs;
695 if (elapsedTimeNs <= 0) {
696 return;
magjeddf494b02016-10-07 05:32:35 -0700697 }
magjed9ab8a182016-10-20 03:18:09 -0700698 final float renderFps = framesRendered * TimeUnit.SECONDS.toNanos(1) / (float) elapsedTimeNs;
699 logD("Duration: " + TimeUnit.NANOSECONDS.toMillis(elapsedTimeNs) + " ms."
700 + " Frames received: " + framesReceived + "."
701 + " Dropped: " + framesDropped + "."
702 + " Rendered: " + framesRendered + "."
Sami Kalliomäki1659e972018-06-04 14:07:58 +0200703 + " Render fps: " + fpsFormat.format(renderFps) + "."
magjed9ab8a182016-10-20 03:18:09 -0700704 + " Average render time: " + averageTimeAsString(renderTimeNs, framesRendered) + "."
705 + " Average swapBuffer time: "
706 + averageTimeAsString(renderSwapBufferTimeNs, framesRendered) + ".");
707 resetStatistics(currentTimeNs);
magjeddf494b02016-10-07 05:32:35 -0700708 }
709 }
710
711 private void logD(String string) {
712 Logging.d(TAG, name + string);
713 }
714}