blob: fc29aa1a418e83f28a00b8c7f87576f97bd6cc82 [file] [log] [blame]
magjeddf494b02016-10-07 05:32:35 -07001/*
2 * Copyright 2016 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11package org.webrtc;
12
sakalfb0c5732016-11-03 09:15:34 -070013import android.graphics.Bitmap;
sakal6bdcefc2017-08-15 01:56:02 -070014import android.graphics.Matrix;
magjed9ab8a182016-10-20 03:18:09 -070015import android.graphics.SurfaceTexture;
magjeddf494b02016-10-07 05:32:35 -070016import android.opengl.GLES20;
17import android.os.Handler;
18import android.os.HandlerThread;
19import android.os.Looper;
20import android.view.Surface;
sakalfb0c5732016-11-03 09:15:34 -070021import java.nio.ByteBuffer;
Sami Kalliomäki1659e972018-06-04 14:07:58 +020022import java.text.DecimalFormat;
sakalfb0c5732016-11-03 09:15:34 -070023import java.util.ArrayList;
24import java.util.Iterator;
magjeddf494b02016-10-07 05:32:35 -070025import java.util.concurrent.CountDownLatch;
26import java.util.concurrent.TimeUnit;
Sami Kalliomäkie7592d82018-03-22 13:32:44 +010027import javax.annotation.Nullable;
magjeddf494b02016-10-07 05:32:35 -070028
29/**
30 * Implements org.webrtc.VideoRenderer.Callbacks by displaying the video stream on an EGL Surface.
31 * This class is intended to be used as a helper class for rendering on SurfaceViews and
32 * TextureViews.
33 */
sakal6bdcefc2017-08-15 01:56:02 -070034public class EglRenderer implements VideoRenderer.Callbacks, VideoSink {
magjeddf494b02016-10-07 05:32:35 -070035 private static final String TAG = "EglRenderer";
magjed9ab8a182016-10-20 03:18:09 -070036 private static final long LOG_INTERVAL_SEC = 4;
magjeddf494b02016-10-07 05:32:35 -070037
sakalfb0c5732016-11-03 09:15:34 -070038 public interface FrameListener { void onFrame(Bitmap frame); }
39
sakal3a9bc172016-11-30 08:30:05 -080040 private static class FrameListenerAndParams {
sakalfb0c5732016-11-03 09:15:34 -070041 public final FrameListener listener;
sakal3a9bc172016-11-30 08:30:05 -080042 public final float scale;
43 public final RendererCommon.GlDrawer drawer;
sakal8fdf9572017-05-31 02:43:10 -070044 public final boolean applyFpsReduction;
sakalfb0c5732016-11-03 09:15:34 -070045
sakal8fdf9572017-05-31 02:43:10 -070046 public FrameListenerAndParams(FrameListener listener, float scale,
47 RendererCommon.GlDrawer drawer, boolean applyFpsReduction) {
sakalfb0c5732016-11-03 09:15:34 -070048 this.listener = listener;
sakal3a9bc172016-11-30 08:30:05 -080049 this.scale = scale;
50 this.drawer = drawer;
sakal8fdf9572017-05-31 02:43:10 -070051 this.applyFpsReduction = applyFpsReduction;
sakalfb0c5732016-11-03 09:15:34 -070052 }
53 }
54
magjeddf494b02016-10-07 05:32:35 -070055 private class EglSurfaceCreation implements Runnable {
magjed9ab8a182016-10-20 03:18:09 -070056 private Object surface;
magjeddf494b02016-10-07 05:32:35 -070057
Mirko Bonadei12251b62017-11-05 19:35:31 -080058 // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression.
59 @SuppressWarnings("NoSynchronizedMethodCheck")
magjed9ab8a182016-10-20 03:18:09 -070060 public synchronized void setSurface(Object surface) {
magjeddf494b02016-10-07 05:32:35 -070061 this.surface = surface;
62 }
63
64 @Override
Mirko Bonadei12251b62017-11-05 19:35:31 -080065 // TODO(bugs.webrtc.org/8491): Remove NoSynchronizedMethodCheck suppression.
66 @SuppressWarnings("NoSynchronizedMethodCheck")
magjeddf494b02016-10-07 05:32:35 -070067 public synchronized void run() {
68 if (surface != null && eglBase != null && !eglBase.hasSurface()) {
magjed9ab8a182016-10-20 03:18:09 -070069 if (surface instanceof Surface) {
70 eglBase.createSurface((Surface) surface);
71 } else if (surface instanceof SurfaceTexture) {
72 eglBase.createSurface((SurfaceTexture) surface);
73 } else {
74 throw new IllegalStateException("Invalid surface: " + surface);
75 }
magjeddf494b02016-10-07 05:32:35 -070076 eglBase.makeCurrent();
77 // Necessary for YUV frames with odd width.
78 GLES20.glPixelStorei(GLES20.GL_UNPACK_ALIGNMENT, 1);
79 }
80 }
81 }
82
Xiaolei Yu149533a2017-11-03 07:55:01 +080083 protected final String name;
magjeddf494b02016-10-07 05:32:35 -070084
85 // |renderThreadHandler| is a handler for communicating with |renderThread|, and is synchronized
86 // on |handlerLock|.
87 private final Object handlerLock = new Object();
Sami Kalliomäkie7592d82018-03-22 13:32:44 +010088 @Nullable private Handler renderThreadHandler;
magjeddf494b02016-10-07 05:32:35 -070089
sakal3a9bc172016-11-30 08:30:05 -080090 private final ArrayList<FrameListenerAndParams> frameListeners = new ArrayList<>();
sakalfb0c5732016-11-03 09:15:34 -070091
magjed9ab8a182016-10-20 03:18:09 -070092 // Variables for fps reduction.
93 private final Object fpsReductionLock = new Object();
94 // Time for when next frame should be rendered.
95 private long nextFrameTimeNs;
96 // Minimum duration between frames when fps reduction is active, or -1 if video is completely
97 // paused.
98 private long minRenderPeriodNs;
99
magjeddf494b02016-10-07 05:32:35 -0700100 // EGL and GL resources for drawing YUV/OES textures. After initilization, these are only accessed
101 // from the render thread.
Sami Kalliomäkie7592d82018-03-22 13:32:44 +0100102 @Nullable private EglBase eglBase;
magjed7cede372017-09-11 06:12:07 -0700103 private final VideoFrameDrawer frameDrawer = new VideoFrameDrawer();
Sami Kalliomäkie7592d82018-03-22 13:32:44 +0100104 @Nullable private RendererCommon.GlDrawer drawer;
magjed7cede372017-09-11 06:12:07 -0700105 private final Matrix drawMatrix = new Matrix();
magjeddf494b02016-10-07 05:32:35 -0700106
107 // Pending frame to render. Serves as a queue with size 1. Synchronized on |frameLock|.
108 private final Object frameLock = new Object();
Sami Kalliomäkie7592d82018-03-22 13:32:44 +0100109 @Nullable private VideoFrame pendingFrame;
magjeddf494b02016-10-07 05:32:35 -0700110
111 // These variables are synchronized on |layoutLock|.
112 private final Object layoutLock = new Object();
magjeddf494b02016-10-07 05:32:35 -0700113 private float layoutAspectRatio;
114 // If true, mirrors the video stream horizontally.
115 private boolean mirror;
116
117 // These variables are synchronized on |statisticsLock|.
118 private final Object statisticsLock = new Object();
119 // Total number of video frames received in renderFrame() call.
120 private int framesReceived;
121 // Number of video frames dropped by renderFrame() because previous frame has not been rendered
122 // yet.
123 private int framesDropped;
124 // Number of rendered video frames.
125 private int framesRendered;
magjed9ab8a182016-10-20 03:18:09 -0700126 // Start time for counting these statistics, or 0 if we haven't started measuring yet.
127 private long statisticsStartTimeNs;
magjeddf494b02016-10-07 05:32:35 -0700128 // Time in ns spent in renderFrameOnRenderThread() function.
129 private long renderTimeNs;
magjed9ab8a182016-10-20 03:18:09 -0700130 // Time in ns spent by the render thread in the swapBuffers() function.
131 private long renderSwapBufferTimeNs;
magjeddf494b02016-10-07 05:32:35 -0700132
sakalfb0c5732016-11-03 09:15:34 -0700133 // Used for bitmap capturing.
Magnus Jedvert2ed62b32018-04-11 14:25:14 +0200134 private final GlTextureFrameBuffer bitmapTextureFramebuffer =
135 new GlTextureFrameBuffer(GLES20.GL_RGBA);
sakalfb0c5732016-11-03 09:15:34 -0700136
magjed9ab8a182016-10-20 03:18:09 -0700137 private final Runnable logStatisticsRunnable = new Runnable() {
138 @Override
139 public void run() {
140 logStatistics();
141 synchronized (handlerLock) {
142 if (renderThreadHandler != null) {
143 renderThreadHandler.removeCallbacks(logStatisticsRunnable);
144 renderThreadHandler.postDelayed(
145 logStatisticsRunnable, TimeUnit.SECONDS.toMillis(LOG_INTERVAL_SEC));
146 }
147 }
148 }
149 };
150
magjeddf494b02016-10-07 05:32:35 -0700151 private final EglSurfaceCreation eglSurfaceCreationRunnable = new EglSurfaceCreation();
152
153 /**
154 * Standard constructor. The name will be used for the render thread name and included when
155 * logging. In order to render something, you must first call init() and createEglSurface.
156 */
157 public EglRenderer(String name) {
158 this.name = name;
159 }
160
161 /**
162 * Initialize this class, sharing resources with |sharedContext|. The custom |drawer| will be used
163 * for drawing frames on the EGLSurface. This class is responsible for calling release() on
164 * |drawer|. It is allowed to call init() to reinitialize the renderer after a previous
165 * init()/release() cycle.
166 */
Sami Kalliomäkie7592d82018-03-22 13:32:44 +0100167 public void init(@Nullable final EglBase.Context sharedContext, final int[] configAttributes,
magjeddf494b02016-10-07 05:32:35 -0700168 RendererCommon.GlDrawer drawer) {
169 synchronized (handlerLock) {
170 if (renderThreadHandler != null) {
171 throw new IllegalStateException(name + "Already initialized");
172 }
173 logD("Initializing EglRenderer");
174 this.drawer = drawer;
175
176 final HandlerThread renderThread = new HandlerThread(name + "EglRenderer");
177 renderThread.start();
178 renderThreadHandler = new Handler(renderThread.getLooper());
179 // Create EGL context on the newly created render thread. It should be possibly to create the
180 // context on this thread and make it current on the render thread, but this causes failure on
181 // some Marvel based JB devices. https://bugs.chromium.org/p/webrtc/issues/detail?id=6350.
sakalbf080602017-08-11 01:42:43 -0700182 ThreadUtils.invokeAtFrontUninterruptibly(renderThreadHandler, () -> {
183 // If sharedContext is null, then texture frames are disabled. This is typically for old
184 // devices that might not be fully spec compliant, so force EGL 1.0 since EGL 1.4 has
185 // caused trouble on some weird devices.
186 if (sharedContext == null) {
187 logD("EglBase10.create context");
188 eglBase = EglBase.createEgl10(configAttributes);
189 } else {
190 logD("EglBase.create shared context");
191 eglBase = EglBase.create(sharedContext, configAttributes);
magjeddf494b02016-10-07 05:32:35 -0700192 }
193 });
magjed9ab8a182016-10-20 03:18:09 -0700194 renderThreadHandler.post(eglSurfaceCreationRunnable);
195 final long currentTimeNs = System.nanoTime();
196 resetStatistics(currentTimeNs);
197 renderThreadHandler.postDelayed(
198 logStatisticsRunnable, TimeUnit.SECONDS.toMillis(LOG_INTERVAL_SEC));
magjeddf494b02016-10-07 05:32:35 -0700199 }
200 }
201
202 public void createEglSurface(Surface surface) {
magjed9ab8a182016-10-20 03:18:09 -0700203 createEglSurfaceInternal(surface);
204 }
205
206 public void createEglSurface(SurfaceTexture surfaceTexture) {
207 createEglSurfaceInternal(surfaceTexture);
208 }
209
210 private void createEglSurfaceInternal(Object surface) {
magjeddf494b02016-10-07 05:32:35 -0700211 eglSurfaceCreationRunnable.setSurface(surface);
magjed9ab8a182016-10-20 03:18:09 -0700212 postToRenderThread(eglSurfaceCreationRunnable);
magjeddf494b02016-10-07 05:32:35 -0700213 }
214
215 /**
216 * Block until any pending frame is returned and all GL resources released, even if an interrupt
217 * occurs. If an interrupt occurs during release(), the interrupt flag will be set. This function
218 * should be called before the Activity is destroyed and the EGLContext is still valid. If you
219 * don't call this function, the GL resources might leak.
220 */
221 public void release() {
magjed9ab8a182016-10-20 03:18:09 -0700222 logD("Releasing.");
magjeddf494b02016-10-07 05:32:35 -0700223 final CountDownLatch eglCleanupBarrier = new CountDownLatch(1);
224 synchronized (handlerLock) {
225 if (renderThreadHandler == null) {
226 logD("Already released");
227 return;
228 }
magjed9ab8a182016-10-20 03:18:09 -0700229 renderThreadHandler.removeCallbacks(logStatisticsRunnable);
magjeddf494b02016-10-07 05:32:35 -0700230 // Release EGL and GL resources on render thread.
sakalbf080602017-08-11 01:42:43 -0700231 renderThreadHandler.postAtFrontOfQueue(() -> {
232 if (drawer != null) {
233 drawer.release();
234 drawer = null;
magjeddf494b02016-10-07 05:32:35 -0700235 }
magjed7cede372017-09-11 06:12:07 -0700236 frameDrawer.release();
Magnus Jedvert2ed62b32018-04-11 14:25:14 +0200237 bitmapTextureFramebuffer.release();
sakalbf080602017-08-11 01:42:43 -0700238 if (eglBase != null) {
239 logD("eglBase detach and release.");
240 eglBase.detachCurrent();
241 eglBase.release();
242 eglBase = null;
243 }
Sami Kalliomäki8ebac242017-11-08 17:13:13 +0100244 frameListeners.clear();
sakalbf080602017-08-11 01:42:43 -0700245 eglCleanupBarrier.countDown();
magjeddf494b02016-10-07 05:32:35 -0700246 });
247 final Looper renderLooper = renderThreadHandler.getLooper();
248 // TODO(magjed): Replace this post() with renderLooper.quitSafely() when API support >= 18.
sakalbf080602017-08-11 01:42:43 -0700249 renderThreadHandler.post(() -> {
250 logD("Quitting render thread.");
251 renderLooper.quit();
magjeddf494b02016-10-07 05:32:35 -0700252 });
253 // Don't accept any more frames or messages to the render thread.
254 renderThreadHandler = null;
255 }
256 // Make sure the EGL/GL cleanup posted above is executed.
257 ThreadUtils.awaitUninterruptibly(eglCleanupBarrier);
258 synchronized (frameLock) {
259 if (pendingFrame != null) {
sakal6bdcefc2017-08-15 01:56:02 -0700260 pendingFrame.release();
magjeddf494b02016-10-07 05:32:35 -0700261 pendingFrame = null;
262 }
263 }
magjeddf494b02016-10-07 05:32:35 -0700264 logD("Releasing done.");
265 }
266
267 /**
magjed9ab8a182016-10-20 03:18:09 -0700268 * Reset the statistics logged in logStatistics().
magjeddf494b02016-10-07 05:32:35 -0700269 */
magjed9ab8a182016-10-20 03:18:09 -0700270 private void resetStatistics(long currentTimeNs) {
magjeddf494b02016-10-07 05:32:35 -0700271 synchronized (statisticsLock) {
magjed9ab8a182016-10-20 03:18:09 -0700272 statisticsStartTimeNs = currentTimeNs;
magjeddf494b02016-10-07 05:32:35 -0700273 framesReceived = 0;
274 framesDropped = 0;
275 framesRendered = 0;
magjeddf494b02016-10-07 05:32:35 -0700276 renderTimeNs = 0;
magjed9ab8a182016-10-20 03:18:09 -0700277 renderSwapBufferTimeNs = 0;
278 }
279 }
280
281 public void printStackTrace() {
282 synchronized (handlerLock) {
283 final Thread renderThread =
284 (renderThreadHandler == null) ? null : renderThreadHandler.getLooper().getThread();
285 if (renderThread != null) {
286 final StackTraceElement[] renderStackTrace = renderThread.getStackTrace();
287 if (renderStackTrace.length > 0) {
288 logD("EglRenderer stack trace:");
289 for (StackTraceElement traceElem : renderStackTrace) {
290 logD(traceElem.toString());
291 }
292 }
293 }
magjeddf494b02016-10-07 05:32:35 -0700294 }
295 }
296
297 /**
298 * Set if the video stream should be mirrored or not.
299 */
300 public void setMirror(final boolean mirror) {
301 logD("setMirror: " + mirror);
302 synchronized (layoutLock) {
303 this.mirror = mirror;
304 }
305 }
306
307 /**
308 * Set layout aspect ratio. This is used to crop frames when rendering to avoid stretched video.
309 * Set this to 0 to disable cropping.
310 */
311 public void setLayoutAspectRatio(float layoutAspectRatio) {
312 logD("setLayoutAspectRatio: " + layoutAspectRatio);
313 synchronized (layoutLock) {
314 this.layoutAspectRatio = layoutAspectRatio;
315 }
316 }
317
magjed9ab8a182016-10-20 03:18:09 -0700318 /**
319 * Limit render framerate.
320 *
321 * @param fps Limit render framerate to this value, or use Float.POSITIVE_INFINITY to disable fps
322 * reduction.
323 */
324 public void setFpsReduction(float fps) {
325 logD("setFpsReduction: " + fps);
326 synchronized (fpsReductionLock) {
327 final long previousRenderPeriodNs = minRenderPeriodNs;
328 if (fps <= 0) {
329 minRenderPeriodNs = Long.MAX_VALUE;
330 } else {
331 minRenderPeriodNs = (long) (TimeUnit.SECONDS.toNanos(1) / fps);
332 }
333 if (minRenderPeriodNs != previousRenderPeriodNs) {
334 // Fps reduction changed - reset frame time.
335 nextFrameTimeNs = System.nanoTime();
336 }
337 }
338 }
339
340 public void disableFpsReduction() {
341 setFpsReduction(Float.POSITIVE_INFINITY /* fps */);
342 }
343
344 public void pauseVideo() {
345 setFpsReduction(0 /* fps */);
346 }
347
sakalfb0c5732016-11-03 09:15:34 -0700348 /**
sakal3a9bc172016-11-30 08:30:05 -0800349 * Register a callback to be invoked when a new video frame has been received. This version uses
350 * the drawer of the EglRenderer that was passed in init.
sakalfb0c5732016-11-03 09:15:34 -0700351 *
sakald1516522017-03-13 05:11:48 -0700352 * @param listener The callback to be invoked. The callback will be invoked on the render thread.
353 * It should be lightweight and must not call removeFrameListener.
sakalfb0c5732016-11-03 09:15:34 -0700354 * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is
355 * required.
356 */
sakalbb584352016-11-28 08:53:44 -0800357 public void addFrameListener(final FrameListener listener, final float scale) {
sakal8fdf9572017-05-31 02:43:10 -0700358 addFrameListener(listener, scale, null, false /* applyFpsReduction */);
sakal3a9bc172016-11-30 08:30:05 -0800359 }
360
361 /**
362 * Register a callback to be invoked when a new video frame has been received.
363 *
sakald1516522017-03-13 05:11:48 -0700364 * @param listener The callback to be invoked. The callback will be invoked on the render thread.
365 * It should be lightweight and must not call removeFrameListener.
sakal3a9bc172016-11-30 08:30:05 -0800366 * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is
367 * required.
sakald1516522017-03-13 05:11:48 -0700368 * @param drawer Custom drawer to use for this frame listener or null to use the default one.
sakal3a9bc172016-11-30 08:30:05 -0800369 */
370 public void addFrameListener(
sakald1516522017-03-13 05:11:48 -0700371 final FrameListener listener, final float scale, final RendererCommon.GlDrawer drawerParam) {
sakal8fdf9572017-05-31 02:43:10 -0700372 addFrameListener(listener, scale, drawerParam, false /* applyFpsReduction */);
373 }
374
375 /**
376 * Register a callback to be invoked when a new video frame has been received.
377 *
378 * @param listener The callback to be invoked. The callback will be invoked on the render thread.
379 * It should be lightweight and must not call removeFrameListener.
380 * @param scale The scale of the Bitmap passed to the callback, or 0 if no Bitmap is
381 * required.
382 * @param drawer Custom drawer to use for this frame listener or null to use the default one.
383 * @param applyFpsReduction This callback will not be called for frames that have been dropped by
384 * FPS reduction.
385 */
386 public void addFrameListener(final FrameListener listener, final float scale,
Sami Kalliomäkie7592d82018-03-22 13:32:44 +0100387 @Nullable final RendererCommon.GlDrawer drawerParam, final boolean applyFpsReduction) {
sakalbf080602017-08-11 01:42:43 -0700388 postToRenderThread(() -> {
389 final RendererCommon.GlDrawer listenerDrawer = drawerParam == null ? drawer : drawerParam;
390 frameListeners.add(
391 new FrameListenerAndParams(listener, scale, listenerDrawer, applyFpsReduction));
sakalbb584352016-11-28 08:53:44 -0800392 });
sakalfb0c5732016-11-03 09:15:34 -0700393 }
394
395 /**
396 * Remove any pending callback that was added with addFrameListener. If the callback is not in
sakalbb584352016-11-28 08:53:44 -0800397 * the queue, nothing happens. It is ensured that callback won't be called after this method
398 * returns.
sakalfb0c5732016-11-03 09:15:34 -0700399 *
400 * @param runnable The callback to remove.
401 */
sakalbb584352016-11-28 08:53:44 -0800402 public void removeFrameListener(final FrameListener listener) {
403 final CountDownLatch latch = new CountDownLatch(1);
Sami Kalliomäki8ebac242017-11-08 17:13:13 +0100404 synchronized (handlerLock) {
405 if (renderThreadHandler == null) {
406 return;
sakalfb0c5732016-11-03 09:15:34 -0700407 }
Sami Kalliomäki8ebac242017-11-08 17:13:13 +0100408 if (Thread.currentThread() == renderThreadHandler.getLooper().getThread()) {
409 throw new RuntimeException("removeFrameListener must not be called on the render thread.");
410 }
411 postToRenderThread(() -> {
412 latch.countDown();
413 final Iterator<FrameListenerAndParams> iter = frameListeners.iterator();
414 while (iter.hasNext()) {
415 if (iter.next().listener == listener) {
416 iter.remove();
417 }
418 }
419 });
420 }
sakalbb584352016-11-28 08:53:44 -0800421 ThreadUtils.awaitUninterruptibly(latch);
sakalfb0c5732016-11-03 09:15:34 -0700422 }
423
magjeddf494b02016-10-07 05:32:35 -0700424 // VideoRenderer.Callbacks interface.
425 @Override
426 public void renderFrame(VideoRenderer.I420Frame frame) {
sakal6bdcefc2017-08-15 01:56:02 -0700427 VideoFrame videoFrame = frame.toVideoFrame();
428 onFrame(videoFrame);
429 videoFrame.release();
430 }
431
432 // VideoSink interface.
433 @Override
434 public void onFrame(VideoFrame frame) {
magjeddf494b02016-10-07 05:32:35 -0700435 synchronized (statisticsLock) {
436 ++framesReceived;
437 }
magjed9ab8a182016-10-20 03:18:09 -0700438 final boolean dropOldFrame;
magjeddf494b02016-10-07 05:32:35 -0700439 synchronized (handlerLock) {
440 if (renderThreadHandler == null) {
441 logD("Dropping frame - Not initialized or already released.");
magjeddf494b02016-10-07 05:32:35 -0700442 return;
443 }
magjed9ab8a182016-10-20 03:18:09 -0700444 synchronized (frameLock) {
445 dropOldFrame = (pendingFrame != null);
446 if (dropOldFrame) {
sakal6bdcefc2017-08-15 01:56:02 -0700447 pendingFrame.release();
magjeddf494b02016-10-07 05:32:35 -0700448 }
449 pendingFrame = frame;
sakal6bdcefc2017-08-15 01:56:02 -0700450 pendingFrame.retain();
sakalbf080602017-08-11 01:42:43 -0700451 renderThreadHandler.post(this ::renderFrameOnRenderThread);
magjeddf494b02016-10-07 05:32:35 -0700452 }
453 }
magjed9ab8a182016-10-20 03:18:09 -0700454 if (dropOldFrame) {
455 synchronized (statisticsLock) {
456 ++framesDropped;
457 }
458 }
magjeddf494b02016-10-07 05:32:35 -0700459 }
460
461 /**
462 * Release EGL surface. This function will block until the EGL surface is released.
463 */
sakal28ec6bd2016-11-09 01:47:12 -0800464 public void releaseEglSurface(final Runnable completionCallback) {
magjeddf494b02016-10-07 05:32:35 -0700465 // Ensure that the render thread is no longer touching the Surface before returning from this
466 // function.
467 eglSurfaceCreationRunnable.setSurface(null /* surface */);
468 synchronized (handlerLock) {
469 if (renderThreadHandler != null) {
470 renderThreadHandler.removeCallbacks(eglSurfaceCreationRunnable);
sakalbf080602017-08-11 01:42:43 -0700471 renderThreadHandler.postAtFrontOfQueue(() -> {
472 if (eglBase != null) {
473 eglBase.detachCurrent();
474 eglBase.releaseSurface();
magjeddf494b02016-10-07 05:32:35 -0700475 }
sakalbf080602017-08-11 01:42:43 -0700476 completionCallback.run();
magjeddf494b02016-10-07 05:32:35 -0700477 });
sakal28ec6bd2016-11-09 01:47:12 -0800478 return;
magjeddf494b02016-10-07 05:32:35 -0700479 }
480 }
sakal28ec6bd2016-11-09 01:47:12 -0800481 completionCallback.run();
magjeddf494b02016-10-07 05:32:35 -0700482 }
483
484 /**
magjeddf494b02016-10-07 05:32:35 -0700485 * Private helper function to post tasks safely.
486 */
magjed9ab8a182016-10-20 03:18:09 -0700487 private void postToRenderThread(Runnable runnable) {
magjeddf494b02016-10-07 05:32:35 -0700488 synchronized (handlerLock) {
489 if (renderThreadHandler != null) {
490 renderThreadHandler.post(runnable);
491 }
492 }
493 }
494
sakalf25a2202017-05-04 06:06:56 -0700495 private void clearSurfaceOnRenderThread(float r, float g, float b, float a) {
magjeddf494b02016-10-07 05:32:35 -0700496 if (eglBase != null && eglBase.hasSurface()) {
497 logD("clearSurface");
sakalf25a2202017-05-04 06:06:56 -0700498 GLES20.glClearColor(r, g, b, a);
magjeddf494b02016-10-07 05:32:35 -0700499 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
500 eglBase.swapBuffers();
501 }
502 }
503
504 /**
sakalf25a2202017-05-04 06:06:56 -0700505 * Post a task to clear the surface to a transparent uniform color.
magjed9ab8a182016-10-20 03:18:09 -0700506 */
507 public void clearImage() {
sakalf25a2202017-05-04 06:06:56 -0700508 clearImage(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */);
509 }
510
511 /**
512 * Post a task to clear the surface to a specific color.
513 */
514 public void clearImage(final float r, final float g, final float b, final float a) {
magjed9ab8a182016-10-20 03:18:09 -0700515 synchronized (handlerLock) {
516 if (renderThreadHandler == null) {
517 return;
518 }
sakalbf080602017-08-11 01:42:43 -0700519 renderThreadHandler.postAtFrontOfQueue(() -> clearSurfaceOnRenderThread(r, g, b, a));
magjed9ab8a182016-10-20 03:18:09 -0700520 }
521 }
522
523 /**
magjeddf494b02016-10-07 05:32:35 -0700524 * Renders and releases |pendingFrame|.
525 */
526 private void renderFrameOnRenderThread() {
527 // Fetch and render |pendingFrame|.
sakal6bdcefc2017-08-15 01:56:02 -0700528 final VideoFrame frame;
magjeddf494b02016-10-07 05:32:35 -0700529 synchronized (frameLock) {
530 if (pendingFrame == null) {
531 return;
532 }
533 frame = pendingFrame;
534 pendingFrame = null;
535 }
536 if (eglBase == null || !eglBase.hasSurface()) {
537 logD("Dropping frame - No surface");
sakal6bdcefc2017-08-15 01:56:02 -0700538 frame.release();
magjeddf494b02016-10-07 05:32:35 -0700539 return;
540 }
sakald1516522017-03-13 05:11:48 -0700541 // Check if fps reduction is active.
542 final boolean shouldRenderFrame;
543 synchronized (fpsReductionLock) {
544 if (minRenderPeriodNs == Long.MAX_VALUE) {
545 // Rendering is paused.
546 shouldRenderFrame = false;
547 } else if (minRenderPeriodNs <= 0) {
548 // FPS reduction is disabled.
549 shouldRenderFrame = true;
550 } else {
551 final long currentTimeNs = System.nanoTime();
552 if (currentTimeNs < nextFrameTimeNs) {
553 logD("Skipping frame rendering - fps reduction is active.");
554 shouldRenderFrame = false;
555 } else {
556 nextFrameTimeNs += minRenderPeriodNs;
557 // The time for the next frame should always be in the future.
558 nextFrameTimeNs = Math.max(nextFrameTimeNs, currentTimeNs);
559 shouldRenderFrame = true;
560 }
561 }
562 }
magjeddf494b02016-10-07 05:32:35 -0700563
564 final long startTimeNs = System.nanoTime();
magjeddf494b02016-10-07 05:32:35 -0700565
sakal6bdcefc2017-08-15 01:56:02 -0700566 final float frameAspectRatio = frame.getRotatedWidth() / (float) frame.getRotatedHeight();
567 final float drawnAspectRatio;
magjeddf494b02016-10-07 05:32:35 -0700568 synchronized (layoutLock) {
sakal6bdcefc2017-08-15 01:56:02 -0700569 drawnAspectRatio = layoutAspectRatio != 0f ? layoutAspectRatio : frameAspectRatio;
magjeddf494b02016-10-07 05:32:35 -0700570 }
571
sakal6bdcefc2017-08-15 01:56:02 -0700572 final float scaleX;
573 final float scaleY;
574
575 if (frameAspectRatio > drawnAspectRatio) {
576 scaleX = drawnAspectRatio / frameAspectRatio;
577 scaleY = 1f;
578 } else {
579 scaleX = 1f;
580 scaleY = frameAspectRatio / drawnAspectRatio;
581 }
582
magjed7cede372017-09-11 06:12:07 -0700583 drawMatrix.reset();
sakal6bdcefc2017-08-15 01:56:02 -0700584 drawMatrix.preTranslate(0.5f, 0.5f);
sakal6bdcefc2017-08-15 01:56:02 -0700585 if (mirror)
586 drawMatrix.preScale(-1f, 1f);
587 drawMatrix.preScale(scaleX, scaleY);
588 drawMatrix.preTranslate(-0.5f, -0.5f);
589
sakald1516522017-03-13 05:11:48 -0700590 if (shouldRenderFrame) {
591 GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */);
592 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
magjed7cede372017-09-11 06:12:07 -0700593 frameDrawer.drawFrame(frame, drawer, drawMatrix, 0 /* viewportX */, 0 /* viewportY */,
594 eglBase.surfaceWidth(), eglBase.surfaceHeight());
sakald1516522017-03-13 05:11:48 -0700595
596 final long swapBuffersStartTimeNs = System.nanoTime();
597 eglBase.swapBuffers();
598
599 final long currentTimeNs = System.nanoTime();
600 synchronized (statisticsLock) {
601 ++framesRendered;
602 renderTimeNs += (currentTimeNs - startTimeNs);
603 renderSwapBufferTimeNs += (currentTimeNs - swapBuffersStartTimeNs);
604 }
magjeddf494b02016-10-07 05:32:35 -0700605 }
606
magjed7cede372017-09-11 06:12:07 -0700607 notifyCallbacks(frame, shouldRenderFrame);
608 frame.release();
sakalfb0c5732016-11-03 09:15:34 -0700609 }
610
magjed7cede372017-09-11 06:12:07 -0700611 private void notifyCallbacks(VideoFrame frame, boolean wasRendered) {
sakalbb584352016-11-28 08:53:44 -0800612 if (frameListeners.isEmpty())
613 return;
sakalfb0c5732016-11-03 09:15:34 -0700614
magjed7cede372017-09-11 06:12:07 -0700615 drawMatrix.reset();
sakal6bdcefc2017-08-15 01:56:02 -0700616 drawMatrix.preTranslate(0.5f, 0.5f);
sakal6bdcefc2017-08-15 01:56:02 -0700617 if (mirror)
618 drawMatrix.preScale(-1f, 1f);
619 drawMatrix.preScale(1f, -1f); // We want the output to be upside down for Bitmap.
620 drawMatrix.preTranslate(-0.5f, -0.5f);
sakalfb0c5732016-11-03 09:15:34 -0700621
sakal8fdf9572017-05-31 02:43:10 -0700622 Iterator<FrameListenerAndParams> it = frameListeners.iterator();
623 while (it.hasNext()) {
624 FrameListenerAndParams listenerAndParams = it.next();
625 if (!wasRendered && listenerAndParams.applyFpsReduction) {
626 continue;
627 }
628 it.remove();
629
sakal6bdcefc2017-08-15 01:56:02 -0700630 final int scaledWidth = (int) (listenerAndParams.scale * frame.getRotatedWidth());
631 final int scaledHeight = (int) (listenerAndParams.scale * frame.getRotatedHeight());
sakalfb0c5732016-11-03 09:15:34 -0700632
633 if (scaledWidth == 0 || scaledHeight == 0) {
sakal3a9bc172016-11-30 08:30:05 -0800634 listenerAndParams.listener.onFrame(null);
sakalfb0c5732016-11-03 09:15:34 -0700635 continue;
636 }
637
sakalfb0c5732016-11-03 09:15:34 -0700638 bitmapTextureFramebuffer.setSize(scaledWidth, scaledHeight);
639
640 GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, bitmapTextureFramebuffer.getFrameBufferId());
641 GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, GLES20.GL_COLOR_ATTACHMENT0,
642 GLES20.GL_TEXTURE_2D, bitmapTextureFramebuffer.getTextureId(), 0);
643
sakal103988d2017-02-17 09:59:01 -0800644 GLES20.glClearColor(0 /* red */, 0 /* green */, 0 /* blue */, 0 /* alpha */);
645 GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);
magjed7cede372017-09-11 06:12:07 -0700646 frameDrawer.drawFrame(frame, listenerAndParams.drawer, drawMatrix, 0 /* viewportX */,
647 0 /* viewportY */, scaledWidth, scaledHeight);
sakalfb0c5732016-11-03 09:15:34 -0700648
649 final ByteBuffer bitmapBuffer = ByteBuffer.allocateDirect(scaledWidth * scaledHeight * 4);
650 GLES20.glViewport(0, 0, scaledWidth, scaledHeight);
651 GLES20.glReadPixels(
652 0, 0, scaledWidth, scaledHeight, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, bitmapBuffer);
653
654 GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0);
655 GlUtil.checkNoGLES2Error("EglRenderer.notifyCallbacks");
656
657 final Bitmap bitmap = Bitmap.createBitmap(scaledWidth, scaledHeight, Bitmap.Config.ARGB_8888);
658 bitmap.copyPixelsFromBuffer(bitmapBuffer);
sakal3a9bc172016-11-30 08:30:05 -0800659 listenerAndParams.listener.onFrame(bitmap);
sakalfb0c5732016-11-03 09:15:34 -0700660 }
magjeddf494b02016-10-07 05:32:35 -0700661 }
662
magjed9ab8a182016-10-20 03:18:09 -0700663 private String averageTimeAsString(long sumTimeNs, int count) {
664 return (count <= 0) ? "NA" : TimeUnit.NANOSECONDS.toMicros(sumTimeNs / count) + " μs";
665 }
666
magjeddf494b02016-10-07 05:32:35 -0700667 private void logStatistics() {
Sami Kalliomäki1659e972018-06-04 14:07:58 +0200668 final DecimalFormat fpsFormat = new DecimalFormat("#.0");
magjed9ab8a182016-10-20 03:18:09 -0700669 final long currentTimeNs = System.nanoTime();
magjeddf494b02016-10-07 05:32:35 -0700670 synchronized (statisticsLock) {
magjed9ab8a182016-10-20 03:18:09 -0700671 final long elapsedTimeNs = currentTimeNs - statisticsStartTimeNs;
672 if (elapsedTimeNs <= 0) {
673 return;
magjeddf494b02016-10-07 05:32:35 -0700674 }
magjed9ab8a182016-10-20 03:18:09 -0700675 final float renderFps = framesRendered * TimeUnit.SECONDS.toNanos(1) / (float) elapsedTimeNs;
676 logD("Duration: " + TimeUnit.NANOSECONDS.toMillis(elapsedTimeNs) + " ms."
677 + " Frames received: " + framesReceived + "."
678 + " Dropped: " + framesDropped + "."
679 + " Rendered: " + framesRendered + "."
Sami Kalliomäki1659e972018-06-04 14:07:58 +0200680 + " Render fps: " + fpsFormat.format(renderFps) + "."
magjed9ab8a182016-10-20 03:18:09 -0700681 + " Average render time: " + averageTimeAsString(renderTimeNs, framesRendered) + "."
682 + " Average swapBuffer time: "
683 + averageTimeAsString(renderSwapBufferTimeNs, framesRendered) + ".");
684 resetStatistics(currentTimeNs);
magjeddf494b02016-10-07 05:32:35 -0700685 }
686 }
687
688 private void logD(String string) {
689 Logging.d(TAG, name + string);
690 }
691}