blob: 4304ff421b42b9b6987cf8138acfb948f391829b [file] [log] [blame]
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +00001/*
kjellanderb24317b2016-02-10 07:54:43 -08002 * Copyright 2014 The WebRTC project authors. All Rights Reserved.
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +00003 *
kjellanderb24317b2016-02-10 07:54:43 -08004 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +00009 */
10
11package org.webrtc;
12
13import android.media.MediaCodec;
buildbot@webrtc.orga09a9992014-08-13 17:26:08 +000014import android.media.MediaCodecInfo;
glaznev@webrtc.org99678452014-09-15 17:52:42 +000015import android.media.MediaCodecInfo.CodecCapabilities;
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +000016import android.media.MediaCodecList;
17import android.media.MediaFormat;
18import android.os.Build;
Per488e75f2015-11-19 10:43:36 +010019import android.os.SystemClock;
glaznev@webrtc.org99678452014-09-15 17:52:42 +000020import android.view.Surface;
21
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +000022import java.nio.ByteBuffer;
Magnus Jedvert7e319372015-10-02 15:49:38 +020023import java.util.Arrays;
Alex Glazneveee86a62016-01-29 14:17:07 -080024import java.util.HashSet;
Per488e75f2015-11-19 10:43:36 +010025import java.util.LinkedList;
Magnus Jedvert91b348c2015-10-07 22:57:06 +020026import java.util.List;
Sami Kalliomakid3235f02016-08-02 15:44:04 +020027import java.util.Queue;
Alex Glazneveee86a62016-01-29 14:17:07 -080028import java.util.Set;
Alex Glaznev5c3da4b2015-10-30 15:31:07 -070029import java.util.concurrent.CountDownLatch;
Per488e75f2015-11-19 10:43:36 +010030import java.util.concurrent.TimeUnit;
magjed8c425aa2015-10-22 16:52:39 -070031
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +000032// Java-side of peerconnection_jni.cc:MediaCodecVideoDecoder.
33// This class is an implementation detail of the Java PeerConnection API.
Patrik Höglund68876f92015-11-12 17:36:48 +010034@SuppressWarnings("deprecation")
Alex Glaznev908e77b2015-04-22 09:25:34 -070035public class MediaCodecVideoDecoder {
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +000036 // This class is constructed, operated, and destroyed by its C++ incarnation,
37 // so the class and its methods have non-public visibility. The API this
38 // class exposes aims to mimic the webrtc::VideoDecoder API as closely as
39 // possibly to minimize the amount of translation work necessary.
40
41 private static final String TAG = "MediaCodecVideoDecoder";
magjedcedddbd2016-02-26 09:36:04 -080042 private static final long MAX_DECODE_TIME_MS = 200;
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +000043
magjed0c29de52017-03-02 00:55:32 -080044 // TODO(magjed): Use MediaFormat constants when part of the public API.
45 private static final String FORMAT_KEY_STRIDE = "stride";
46 private static final String FORMAT_KEY_SLICE_HEIGHT = "slice-height";
47 private static final String FORMAT_KEY_CROP_LEFT = "crop-left";
48 private static final String FORMAT_KEY_CROP_RIGHT = "crop-right";
49 private static final String FORMAT_KEY_CROP_TOP = "crop-top";
50 private static final String FORMAT_KEY_CROP_BOTTOM = "crop-bottom";
51
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +000052 // Tracks webrtc::VideoCodecType.
sakalb6760f92016-09-29 04:12:44 -070053 public enum VideoCodecType { VIDEO_CODEC_VP8, VIDEO_CODEC_VP9, VIDEO_CODEC_H264 }
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +000054
Alex Glazneveee86a62016-01-29 14:17:07 -080055 // Timeout for input buffer dequeue.
56 private static final int DEQUEUE_INPUT_TIMEOUT = 500000;
57 // Timeout for codec releasing.
58 private static final int MEDIA_CODEC_RELEASE_TIMEOUT_MS = 5000;
59 // Max number of output buffers queued before starting to drop decoded frames.
60 private static final int MAX_QUEUED_OUTPUTBUFFERS = 3;
Alex Glaznevc6aec4b2015-10-19 16:39:19 -070061 // Active running decoder instance. Set in initDecode() (called from native code)
62 // and reset to null in release() call.
63 private static MediaCodecVideoDecoder runningInstance = null;
Alex Glaznev5c3da4b2015-10-30 15:31:07 -070064 private static MediaCodecVideoDecoderErrorCallback errorCallback = null;
65 private static int codecErrors = 0;
Alex Glazneveee86a62016-01-29 14:17:07 -080066 // List of disabled codec types - can be set from application.
67 private static Set<String> hwDecoderDisabledTypes = new HashSet<String>();
Alex Glaznev5c3da4b2015-10-30 15:31:07 -070068
Alejandro Luebs69ddaef2015-10-09 15:46:09 -070069 private Thread mediaCodecThread;
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +000070 private MediaCodec mediaCodec;
71 private ByteBuffer[] inputBuffers;
72 private ByteBuffer[] outputBuffers;
73 private static final String VP8_MIME_TYPE = "video/x-vnd.on2.vp8";
Alex Glaznev69a7fd52015-11-10 10:25:40 -080074 private static final String VP9_MIME_TYPE = "video/x-vnd.on2.vp9";
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +000075 private static final String H264_MIME_TYPE = "video/avc";
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +000076 // List of supported HW VP8 decoders.
sakalb6760f92016-09-29 04:12:44 -070077 private static final String[] supportedVp8HwCodecPrefixes = {
78 "OMX.qcom.", "OMX.Nvidia.", "OMX.Exynos.", "OMX.Intel."};
Alex Glaznev69a7fd52015-11-10 10:25:40 -080079 // List of supported HW VP9 decoders.
sakalb6760f92016-09-29 04:12:44 -070080 private static final String[] supportedVp9HwCodecPrefixes = {"OMX.qcom.", "OMX.Exynos."};
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +000081 // List of supported HW H.264 decoders.
sakalb6760f92016-09-29 04:12:44 -070082 private static final String[] supportedH264HwCodecPrefixes = {
83 "OMX.qcom.", "OMX.Intel.", "OMX.Exynos."};
glaznev0c1d0602017-01-27 12:24:24 -080084 // List of supported HW H.264 high profile decoders.
glaznevcca0f6c2017-06-14 10:20:54 -070085 private static final String supportedQcomH264HighProfileHwCodecPrefix = "OMX.qcom.";
86 private static final String supportedExynosH264HighProfileHwCodecPrefix = "OMX.Exynos.";
glaznev893a7ee2016-09-22 10:44:30 -070087
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +000088 // NV12 color format supported by QCOM codec, but not declared in MediaCodec -
89 // see /hardware/qcom/media/mm-core/inc/OMX_QCOMExtns.h
glaznev893a7ee2016-09-22 10:44:30 -070090 private static final int COLOR_QCOM_FORMATYVU420PackedSemiPlanar32m4ka = 0x7FA30C01;
91 private static final int COLOR_QCOM_FORMATYVU420PackedSemiPlanar16m4ka = 0x7FA30C02;
92 private static final int COLOR_QCOM_FORMATYVU420PackedSemiPlanar64x32Tile2m8ka = 0x7FA30C03;
93 private static final int COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m = 0x7FA30C04;
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +000094 // Allowable color formats supported by codec - in order of preference.
Magnus Jedvert7e319372015-10-02 15:49:38 +020095 private static final List<Integer> supportedColorList = Arrays.asList(
sakalb6760f92016-09-29 04:12:44 -070096 CodecCapabilities.COLOR_FormatYUV420Planar, CodecCapabilities.COLOR_FormatYUV420SemiPlanar,
97 CodecCapabilities.COLOR_QCOM_FormatYUV420SemiPlanar,
98 COLOR_QCOM_FORMATYVU420PackedSemiPlanar32m4ka, COLOR_QCOM_FORMATYVU420PackedSemiPlanar16m4ka,
99 COLOR_QCOM_FORMATYVU420PackedSemiPlanar64x32Tile2m8ka,
100 COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m);
glaznev893a7ee2016-09-22 10:44:30 -0700101
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000102 private int colorFormat;
103 private int width;
104 private int height;
105 private int stride;
106 private int sliceHeight;
Per488e75f2015-11-19 10:43:36 +0100107 private boolean hasDecodedFirstFrame;
108 private final Queue<TimeStamps> decodeStartTimeMs = new LinkedList<TimeStamps>();
glaznev@webrtc.org99678452014-09-15 17:52:42 +0000109 private boolean useSurface;
Perc01c2542015-11-13 16:58:26 +0100110
Per488e75f2015-11-19 10:43:36 +0100111 // The below variables are only used when decoding to a Surface.
112 private TextureListener textureListener;
Per488e75f2015-11-19 10:43:36 +0100113 private int droppedFrames;
114 private Surface surface = null;
sakalb6760f92016-09-29 04:12:44 -0700115 private final Queue<DecodedOutputBuffer> dequeuedSurfaceOutputBuffers =
116 new LinkedList<DecodedOutputBuffer>();
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000117
Alex Glaznev5c3da4b2015-10-30 15:31:07 -0700118 // MediaCodec error handler - invoked when critical error happens which may prevent
119 // further use of media codec API. Now it means that one of media codec instances
120 // is hanging and can no longer be used in the next call.
121 public static interface MediaCodecVideoDecoderErrorCallback {
122 void onMediaCodecVideoDecoderCriticalError(int codecErrors);
123 }
124
125 public static void setErrorCallback(MediaCodecVideoDecoderErrorCallback errorCallback) {
126 Logging.d(TAG, "Set error callback");
127 MediaCodecVideoDecoder.errorCallback = errorCallback;
128 }
129
Alex Glazneveee86a62016-01-29 14:17:07 -0800130 // Functions to disable HW decoding - can be called from applications for platforms
131 // which have known HW decoding problems.
132 public static void disableVp8HwCodec() {
133 Logging.w(TAG, "VP8 decoding is disabled by application.");
134 hwDecoderDisabledTypes.add(VP8_MIME_TYPE);
135 }
136
137 public static void disableVp9HwCodec() {
138 Logging.w(TAG, "VP9 decoding is disabled by application.");
139 hwDecoderDisabledTypes.add(VP9_MIME_TYPE);
140 }
141
142 public static void disableH264HwCodec() {
143 Logging.w(TAG, "H.264 decoding is disabled by application.");
144 hwDecoderDisabledTypes.add(H264_MIME_TYPE);
145 }
146
147 // Functions to query if HW decoding is supported.
148 public static boolean isVp8HwSupported() {
sakalb6760f92016-09-29 04:12:44 -0700149 return !hwDecoderDisabledTypes.contains(VP8_MIME_TYPE)
150 && (findDecoder(VP8_MIME_TYPE, supportedVp8HwCodecPrefixes) != null);
Alex Glazneveee86a62016-01-29 14:17:07 -0800151 }
152
153 public static boolean isVp9HwSupported() {
sakalb6760f92016-09-29 04:12:44 -0700154 return !hwDecoderDisabledTypes.contains(VP9_MIME_TYPE)
155 && (findDecoder(VP9_MIME_TYPE, supportedVp9HwCodecPrefixes) != null);
Alex Glazneveee86a62016-01-29 14:17:07 -0800156 }
157
158 public static boolean isH264HwSupported() {
sakalb6760f92016-09-29 04:12:44 -0700159 return !hwDecoderDisabledTypes.contains(H264_MIME_TYPE)
160 && (findDecoder(H264_MIME_TYPE, supportedH264HwCodecPrefixes) != null);
Alex Glazneveee86a62016-01-29 14:17:07 -0800161 }
162
glaznev0c1d0602017-01-27 12:24:24 -0800163 public static boolean isH264HighProfileHwSupported() {
glaznevcca0f6c2017-06-14 10:20:54 -0700164 if (hwDecoderDisabledTypes.contains(H264_MIME_TYPE)) {
165 return false;
166 }
167 // Support H.264 HP decoding on QCOM chips for Android L and above.
168 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP
169 && findDecoder(H264_MIME_TYPE, new String[] {supportedQcomH264HighProfileHwCodecPrefix})
170 != null) {
171 return true;
172 }
173 // Support H.264 HP decoding on Exynos chips for Android M and above.
174 if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M
175 && findDecoder(H264_MIME_TYPE, new String[] {supportedExynosH264HighProfileHwCodecPrefix})
176 != null) {
177 return true;
178 }
179 return false;
glaznev0c1d0602017-01-27 12:24:24 -0800180 }
181
Alex Glazneveee86a62016-01-29 14:17:07 -0800182 public static void printStackTrace() {
183 if (runningInstance != null && runningInstance.mediaCodecThread != null) {
184 StackTraceElement[] mediaCodecStackTraces = runningInstance.mediaCodecThread.getStackTrace();
185 if (mediaCodecStackTraces.length > 0) {
186 Logging.d(TAG, "MediaCodecVideoDecoder stacks trace:");
187 for (StackTraceElement stackTrace : mediaCodecStackTraces) {
188 Logging.d(TAG, stackTrace.toString());
189 }
190 }
191 }
192 }
193
194 // Helper struct for findDecoder() below.
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000195 private static class DecoderProperties {
glaznev@webrtc.org99678452014-09-15 17:52:42 +0000196 public DecoderProperties(String codecName, int colorFormat) {
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000197 this.codecName = codecName;
198 this.colorFormat = colorFormat;
199 }
200 public final String codecName; // OpenMax component name for VP8 codec.
sakalb6760f92016-09-29 04:12:44 -0700201 public final int colorFormat; // Color format supported by codec.
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000202 }
203
sakalb6760f92016-09-29 04:12:44 -0700204 private static DecoderProperties findDecoder(String mime, String[] supportedCodecPrefixes) {
glaznev@webrtc.org25cc7452014-10-02 16:58:05 +0000205 if (Build.VERSION.SDK_INT < Build.VERSION_CODES.KITKAT) {
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000206 return null; // MediaCodec.setParameters is missing.
glaznev@webrtc.org25cc7452014-10-02 16:58:05 +0000207 }
Alex Glaznev69a7fd52015-11-10 10:25:40 -0800208 Logging.d(TAG, "Trying to find HW decoder for mime " + mime);
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000209 for (int i = 0; i < MediaCodecList.getCodecCount(); ++i) {
Alex Glaznev0060c562016-08-08 12:27:24 -0700210 MediaCodecInfo info = null;
211 try {
212 info = MediaCodecList.getCodecInfoAt(i);
213 } catch (IllegalArgumentException e) {
sakalb6760f92016-09-29 04:12:44 -0700214 Logging.e(TAG, "Cannot retrieve decoder codec info", e);
Alex Glaznev0060c562016-08-08 12:27:24 -0700215 }
216 if (info == null || info.isEncoder()) {
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000217 continue;
218 }
219 String name = null;
220 for (String mimeType : info.getSupportedTypes()) {
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000221 if (mimeType.equals(mime)) {
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000222 name = info.getName();
223 break;
224 }
225 }
226 if (name == null) {
sakalb6760f92016-09-29 04:12:44 -0700227 continue; // No HW support in this codec; try the next one.
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000228 }
Alex Glaznev69a7fd52015-11-10 10:25:40 -0800229 Logging.d(TAG, "Found candidate decoder " + name);
glaznev@webrtc.org99678452014-09-15 17:52:42 +0000230
glaznev@webrtc.org25cc7452014-10-02 16:58:05 +0000231 // Check if this is supported decoder.
glaznev@webrtc.org99678452014-09-15 17:52:42 +0000232 boolean supportedCodec = false;
glaznev@webrtc.org25cc7452014-10-02 16:58:05 +0000233 for (String codecPrefix : supportedCodecPrefixes) {
234 if (name.startsWith(codecPrefix)) {
glaznev@webrtc.org99678452014-09-15 17:52:42 +0000235 supportedCodec = true;
236 break;
237 }
238 }
239 if (!supportedCodec) {
240 continue;
241 }
242
243 // Check if codec supports either yuv420 or nv12.
Alex Glaznev0060c562016-08-08 12:27:24 -0700244 CodecCapabilities capabilities;
245 try {
246 capabilities = info.getCapabilitiesForType(mime);
247 } catch (IllegalArgumentException e) {
sakalb6760f92016-09-29 04:12:44 -0700248 Logging.e(TAG, "Cannot retrieve decoder capabilities", e);
Alex Glaznev0060c562016-08-08 12:27:24 -0700249 continue;
250 }
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000251 for (int colorFormat : capabilities.colorFormats) {
Jiayang Liu5975b3c2015-09-16 13:40:53 -0700252 Logging.v(TAG, " Color: 0x" + Integer.toHexString(colorFormat));
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000253 }
glaznev@webrtc.org99678452014-09-15 17:52:42 +0000254 for (int supportedColorFormat : supportedColorList) {
255 for (int codecColorFormat : capabilities.colorFormats) {
256 if (codecColorFormat == supportedColorFormat) {
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000257 // Found supported HW decoder.
sakalb6760f92016-09-29 04:12:44 -0700258 Logging.d(TAG, "Found target decoder " + name + ". Color: 0x"
259 + Integer.toHexString(codecColorFormat));
glaznev@webrtc.org99678452014-09-15 17:52:42 +0000260 return new DecoderProperties(name, codecColorFormat);
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000261 }
262 }
263 }
264 }
Alex Glaznev69a7fd52015-11-10 10:25:40 -0800265 Logging.d(TAG, "No HW decoder found for mime " + mime);
sakalb6760f92016-09-29 04:12:44 -0700266 return null; // No HW decoder.
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000267 }
268
Magnus Jedvert7e319372015-10-02 15:49:38 +0200269 private void checkOnMediaCodecThread() throws IllegalStateException {
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000270 if (mediaCodecThread.getId() != Thread.currentThread().getId()) {
sakalb6760f92016-09-29 04:12:44 -0700271 throw new IllegalStateException("MediaCodecVideoDecoder previously operated on "
272 + mediaCodecThread + " but is now called on " + Thread.currentThread());
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000273 }
274 }
275
Per488e75f2015-11-19 10:43:36 +0100276 // Pass null in |surfaceTextureHelper| to configure the codec for ByteBuffer output.
277 private boolean initDecode(
sakalb6760f92016-09-29 04:12:44 -0700278 VideoCodecType type, int width, int height, SurfaceTextureHelper surfaceTextureHelper) {
Alejandro Luebs69ddaef2015-10-09 15:46:09 -0700279 if (mediaCodecThread != null) {
Alex Glaznev6a4a03c2016-03-04 14:10:50 -0800280 throw new RuntimeException("initDecode: Forgot to release()?");
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000281 }
Alex Glaznev6a4a03c2016-03-04 14:10:50 -0800282
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000283 String mime = null;
Alex Glaznev6a4a03c2016-03-04 14:10:50 -0800284 useSurface = (surfaceTextureHelper != null);
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000285 String[] supportedCodecPrefixes = null;
286 if (type == VideoCodecType.VIDEO_CODEC_VP8) {
287 mime = VP8_MIME_TYPE;
288 supportedCodecPrefixes = supportedVp8HwCodecPrefixes;
Alex Glaznev69a7fd52015-11-10 10:25:40 -0800289 } else if (type == VideoCodecType.VIDEO_CODEC_VP9) {
290 mime = VP9_MIME_TYPE;
291 supportedCodecPrefixes = supportedVp9HwCodecPrefixes;
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000292 } else if (type == VideoCodecType.VIDEO_CODEC_H264) {
293 mime = H264_MIME_TYPE;
294 supportedCodecPrefixes = supportedH264HwCodecPrefixes;
295 } else {
Alex Glaznev6a4a03c2016-03-04 14:10:50 -0800296 throw new RuntimeException("initDecode: Non-supported codec " + type);
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000297 }
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000298 DecoderProperties properties = findDecoder(mime, supportedCodecPrefixes);
299 if (properties == null) {
300 throw new RuntimeException("Cannot find HW decoder for " + type);
301 }
Alex Glaznev6a4a03c2016-03-04 14:10:50 -0800302
sakalb6760f92016-09-29 04:12:44 -0700303 Logging.d(TAG, "Java initDecode: " + type + " : " + width + " x " + height + ". Color: 0x"
304 + Integer.toHexString(properties.colorFormat) + ". Use Surface: " + useSurface);
Alex Glaznev6a4a03c2016-03-04 14:10:50 -0800305
Alex Glaznevc6aec4b2015-10-19 16:39:19 -0700306 runningInstance = this; // Decoder is now running and can be queried for stack traces.
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000307 mediaCodecThread = Thread.currentThread();
308 try {
309 this.width = width;
310 this.height = height;
311 stride = width;
312 sliceHeight = height;
glaznev@webrtc.org99678452014-09-15 17:52:42 +0000313
314 if (useSurface) {
Per488e75f2015-11-19 10:43:36 +0100315 textureListener = new TextureListener(surfaceTextureHelper);
316 surface = new Surface(surfaceTextureHelper.getSurfaceTexture());
Magnus Jedvert80cf97c2015-06-11 10:08:59 +0200317 }
glaznev@webrtc.org99678452014-09-15 17:52:42 +0000318
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000319 MediaFormat format = MediaFormat.createVideoFormat(mime, width, height);
glaznev@webrtc.org99678452014-09-15 17:52:42 +0000320 if (!useSurface) {
321 format.setInteger(MediaFormat.KEY_COLOR_FORMAT, properties.colorFormat);
322 }
Jiayang Liu5975b3c2015-09-16 13:40:53 -0700323 Logging.d(TAG, " Format: " + format);
Alex Glaznev6a4a03c2016-03-04 14:10:50 -0800324 mediaCodec = MediaCodecVideoEncoder.createByCodecName(properties.codecName);
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000325 if (mediaCodec == null) {
Alex Glaznev325d4142015-10-12 14:56:02 -0700326 Logging.e(TAG, "Can not create media decoder");
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000327 return false;
328 }
Magnus Jedvert7e319372015-10-02 15:49:38 +0200329 mediaCodec.configure(format, surface, null, 0);
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000330 mediaCodec.start();
Alex Glaznev6a4a03c2016-03-04 14:10:50 -0800331
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000332 colorFormat = properties.colorFormat;
333 outputBuffers = mediaCodec.getOutputBuffers();
334 inputBuffers = mediaCodec.getInputBuffers();
Per488e75f2015-11-19 10:43:36 +0100335 decodeStartTimeMs.clear();
336 hasDecodedFirstFrame = false;
337 dequeuedSurfaceOutputBuffers.clear();
338 droppedFrames = 0;
sakalb6760f92016-09-29 04:12:44 -0700339 Logging.d(TAG,
340 "Input buffers: " + inputBuffers.length + ". Output buffers: " + outputBuffers.length);
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000341 return true;
342 } catch (IllegalStateException e) {
Jiayang Liu5975b3c2015-09-16 13:40:53 -0700343 Logging.e(TAG, "initDecode failed", e);
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000344 return false;
345 }
346 }
347
Alex Glaznev6a4a03c2016-03-04 14:10:50 -0800348 // Resets the decoder so it can start decoding frames with new resolution.
349 // Flushes MediaCodec and clears decoder output buffers.
350 private void reset(int width, int height) {
351 if (mediaCodecThread == null || mediaCodec == null) {
352 throw new RuntimeException("Incorrect reset call for non-initialized decoder.");
353 }
354 Logging.d(TAG, "Java reset: " + width + " x " + height);
355
356 mediaCodec.flush();
357
358 this.width = width;
359 this.height = height;
360 decodeStartTimeMs.clear();
361 dequeuedSurfaceOutputBuffers.clear();
362 hasDecodedFirstFrame = false;
363 droppedFrames = 0;
364 }
365
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000366 private void release() {
Per488e75f2015-11-19 10:43:36 +0100367 Logging.d(TAG, "Java releaseDecoder. Total number of dropped frames: " + droppedFrames);
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000368 checkOnMediaCodecThread();
Alex Glaznev5c3da4b2015-10-30 15:31:07 -0700369
370 // Run Mediacodec stop() and release() on separate thread since sometime
371 // Mediacodec.stop() may hang.
372 final CountDownLatch releaseDone = new CountDownLatch(1);
373
374 Runnable runMediaCodecRelease = new Runnable() {
375 @Override
376 public void run() {
377 try {
378 Logging.d(TAG, "Java releaseDecoder on release thread");
379 mediaCodec.stop();
380 mediaCodec.release();
381 Logging.d(TAG, "Java releaseDecoder on release thread done");
382 } catch (Exception e) {
383 Logging.e(TAG, "Media decoder release failed", e);
384 }
385 releaseDone.countDown();
386 }
387 };
388 new Thread(runMediaCodecRelease).start();
389
390 if (!ThreadUtils.awaitUninterruptibly(releaseDone, MEDIA_CODEC_RELEASE_TIMEOUT_MS)) {
391 Logging.e(TAG, "Media decoder release timeout");
392 codecErrors++;
393 if (errorCallback != null) {
394 Logging.e(TAG, "Invoke codec error callback. Errors: " + codecErrors);
395 errorCallback.onMediaCodecVideoDecoderCriticalError(codecErrors);
396 }
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000397 }
Alex Glaznev5c3da4b2015-10-30 15:31:07 -0700398
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000399 mediaCodec = null;
400 mediaCodecThread = null;
Alex Glaznevc6aec4b2015-10-19 16:39:19 -0700401 runningInstance = null;
glaznev@webrtc.org99678452014-09-15 17:52:42 +0000402 if (useSurface) {
403 surface.release();
Magnus Jedvert7e319372015-10-02 15:49:38 +0200404 surface = null;
Per488e75f2015-11-19 10:43:36 +0100405 textureListener.release();
glaznev@webrtc.org99678452014-09-15 17:52:42 +0000406 }
Alex Glaznev325d4142015-10-12 14:56:02 -0700407 Logging.d(TAG, "Java releaseDecoder done");
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000408 }
409
410 // Dequeue an input buffer and return its index, -1 if no input buffer is
411 // available, or -2 if the codec is no longer operative.
412 private int dequeueInputBuffer() {
413 checkOnMediaCodecThread();
414 try {
glaznev@webrtc.org99678452014-09-15 17:52:42 +0000415 return mediaCodec.dequeueInputBuffer(DEQUEUE_INPUT_TIMEOUT);
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000416 } catch (IllegalStateException e) {
Jiayang Liu5975b3c2015-09-16 13:40:53 -0700417 Logging.e(TAG, "dequeueIntputBuffer failed", e);
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000418 return -2;
419 }
420 }
421
Per488e75f2015-11-19 10:43:36 +0100422 private boolean queueInputBuffer(int inputBufferIndex, int size, long presentationTimeStamUs,
423 long timeStampMs, long ntpTimeStamp) {
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000424 checkOnMediaCodecThread();
425 try {
426 inputBuffers[inputBufferIndex].position(0);
427 inputBuffers[inputBufferIndex].limit(size);
sakalb6760f92016-09-29 04:12:44 -0700428 decodeStartTimeMs.add(
429 new TimeStamps(SystemClock.elapsedRealtime(), timeStampMs, ntpTimeStamp));
Per488e75f2015-11-19 10:43:36 +0100430 mediaCodec.queueInputBuffer(inputBufferIndex, 0, size, presentationTimeStamUs, 0);
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000431 return true;
sakalb6760f92016-09-29 04:12:44 -0700432 } catch (IllegalStateException e) {
Jiayang Liu5975b3c2015-09-16 13:40:53 -0700433 Logging.e(TAG, "decode failed", e);
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000434 return false;
435 }
436 }
437
Per488e75f2015-11-19 10:43:36 +0100438 private static class TimeStamps {
439 public TimeStamps(long decodeStartTimeMs, long timeStampMs, long ntpTimeStampMs) {
440 this.decodeStartTimeMs = decodeStartTimeMs;
441 this.timeStampMs = timeStampMs;
442 this.ntpTimeStampMs = ntpTimeStampMs;
443 }
Alex Glazneveee86a62016-01-29 14:17:07 -0800444 // Time when this frame was queued for decoding.
445 private final long decodeStartTimeMs;
446 // Only used for bookkeeping in Java. Stores C++ inputImage._timeStamp value for input frame.
447 private final long timeStampMs;
448 // Only used for bookkeeping in Java. Stores C++ inputImage.ntp_time_ms_ value for input frame.
449 private final long ntpTimeStampMs;
Per488e75f2015-11-19 10:43:36 +0100450 }
451
452 // Helper struct for dequeueOutputBuffer() below.
453 private static class DecodedOutputBuffer {
glaznev94291482016-02-01 13:17:18 -0800454 public DecodedOutputBuffer(int index, int offset, int size, long presentationTimeStampMs,
455 long timeStampMs, long ntpTimeStampMs, long decodeTime, long endDecodeTime) {
glaznev@webrtc.org99678452014-09-15 17:52:42 +0000456 this.index = index;
457 this.offset = offset;
458 this.size = size;
glaznev94291482016-02-01 13:17:18 -0800459 this.presentationTimeStampMs = presentationTimeStampMs;
Per488e75f2015-11-19 10:43:36 +0100460 this.timeStampMs = timeStampMs;
461 this.ntpTimeStampMs = ntpTimeStampMs;
462 this.decodeTimeMs = decodeTime;
463 this.endDecodeTimeMs = endDecodeTime;
glaznev@webrtc.org99678452014-09-15 17:52:42 +0000464 }
465
466 private final int index;
467 private final int offset;
468 private final int size;
glaznev94291482016-02-01 13:17:18 -0800469 // Presentation timestamp returned in dequeueOutputBuffer call.
470 private final long presentationTimeStampMs;
Alex Glazneveee86a62016-01-29 14:17:07 -0800471 // C++ inputImage._timeStamp value for output frame.
Per488e75f2015-11-19 10:43:36 +0100472 private final long timeStampMs;
Alex Glazneveee86a62016-01-29 14:17:07 -0800473 // C++ inputImage.ntp_time_ms_ value for output frame.
Per488e75f2015-11-19 10:43:36 +0100474 private final long ntpTimeStampMs;
475 // Number of ms it took to decode this frame.
476 private final long decodeTimeMs;
Alex Glazneveee86a62016-01-29 14:17:07 -0800477 // System time when this frame decoding finished.
Per488e75f2015-11-19 10:43:36 +0100478 private final long endDecodeTimeMs;
glaznev@webrtc.org99678452014-09-15 17:52:42 +0000479 }
480
Per488e75f2015-11-19 10:43:36 +0100481 // Helper struct for dequeueTextureBuffer() below.
magjed44bf6f52015-10-03 02:08:00 -0700482 private static class DecodedTextureBuffer {
483 private final int textureID;
Per488e75f2015-11-19 10:43:36 +0100484 private final float[] transformMatrix;
glaznev94291482016-02-01 13:17:18 -0800485 // Presentation timestamp returned in dequeueOutputBuffer call.
486 private final long presentationTimeStampMs;
Alex Glazneveee86a62016-01-29 14:17:07 -0800487 // C++ inputImage._timeStamp value for output frame.
Per488e75f2015-11-19 10:43:36 +0100488 private final long timeStampMs;
Alex Glazneveee86a62016-01-29 14:17:07 -0800489 // C++ inputImage.ntp_time_ms_ value for output frame.
Per488e75f2015-11-19 10:43:36 +0100490 private final long ntpTimeStampMs;
Alex Glazneveee86a62016-01-29 14:17:07 -0800491 // Number of ms it took to decode this frame.
Per488e75f2015-11-19 10:43:36 +0100492 private final long decodeTimeMs;
493 // Interval from when the frame finished decoding until this buffer has been created.
494 // Since there is only one texture, this interval depend on the time from when
495 // a frame is decoded and provided to C++ and until that frame is returned to the MediaCodec
496 // so that the texture can be updated with the next decoded frame.
497 private final long frameDelayMs;
magjed44bf6f52015-10-03 02:08:00 -0700498
Per488e75f2015-11-19 10:43:36 +0100499 // A DecodedTextureBuffer with zero |textureID| has special meaning and represents a frame
500 // that was dropped.
glaznev94291482016-02-01 13:17:18 -0800501 public DecodedTextureBuffer(int textureID, float[] transformMatrix,
502 long presentationTimeStampMs, long timeStampMs, long ntpTimeStampMs, long decodeTimeMs,
503 long frameDelay) {
magjed44bf6f52015-10-03 02:08:00 -0700504 this.textureID = textureID;
Per488e75f2015-11-19 10:43:36 +0100505 this.transformMatrix = transformMatrix;
glaznev94291482016-02-01 13:17:18 -0800506 this.presentationTimeStampMs = presentationTimeStampMs;
Per488e75f2015-11-19 10:43:36 +0100507 this.timeStampMs = timeStampMs;
508 this.ntpTimeStampMs = ntpTimeStampMs;
509 this.decodeTimeMs = decodeTimeMs;
510 this.frameDelayMs = frameDelay;
magjed44bf6f52015-10-03 02:08:00 -0700511 }
512 }
513
Per488e75f2015-11-19 10:43:36 +0100514 // Poll based texture listener.
515 private static class TextureListener
516 implements SurfaceTextureHelper.OnTextureFrameAvailableListener {
517 private final SurfaceTextureHelper surfaceTextureHelper;
518 // |newFrameLock| is used to synchronize arrival of new frames with wait()/notifyAll().
519 private final Object newFrameLock = new Object();
520 // |bufferToRender| is non-null when waiting for transition between addBufferToRender() to
521 // onTextureFrameAvailable().
522 private DecodedOutputBuffer bufferToRender;
523 private DecodedTextureBuffer renderedBuffer;
524
525 public TextureListener(SurfaceTextureHelper surfaceTextureHelper) {
526 this.surfaceTextureHelper = surfaceTextureHelper;
magjed81e8e372016-03-03 02:11:44 -0800527 surfaceTextureHelper.startListening(this);
Per488e75f2015-11-19 10:43:36 +0100528 }
529
530 public void addBufferToRender(DecodedOutputBuffer buffer) {
531 if (bufferToRender != null) {
sakalb6760f92016-09-29 04:12:44 -0700532 Logging.e(TAG, "Unexpected addBufferToRender() called while waiting for a texture.");
Per488e75f2015-11-19 10:43:36 +0100533 throw new IllegalStateException("Waiting for a texture.");
534 }
535 bufferToRender = buffer;
536 }
537
538 public boolean isWaitingForTexture() {
539 synchronized (newFrameLock) {
540 return bufferToRender != null;
541 }
542 }
543
544 // Callback from |surfaceTextureHelper|. May be called on an arbitrary thread.
545 @Override
546 public void onTextureFrameAvailable(
547 int oesTextureId, float[] transformMatrix, long timestampNs) {
548 synchronized (newFrameLock) {
549 if (renderedBuffer != null) {
sakalb6760f92016-09-29 04:12:44 -0700550 Logging.e(
551 TAG, "Unexpected onTextureFrameAvailable() called while already holding a texture.");
Per488e75f2015-11-19 10:43:36 +0100552 throw new IllegalStateException("Already holding a texture.");
553 }
554 // |timestampNs| is always zero on some Android versions.
555 renderedBuffer = new DecodedTextureBuffer(oesTextureId, transformMatrix,
glaznev94291482016-02-01 13:17:18 -0800556 bufferToRender.presentationTimeStampMs, bufferToRender.timeStampMs,
557 bufferToRender.ntpTimeStampMs, bufferToRender.decodeTimeMs,
Per488e75f2015-11-19 10:43:36 +0100558 SystemClock.elapsedRealtime() - bufferToRender.endDecodeTimeMs);
559 bufferToRender = null;
560 newFrameLock.notifyAll();
561 }
562 }
563
564 // Dequeues and returns a DecodedTextureBuffer if available, or null otherwise.
565 public DecodedTextureBuffer dequeueTextureBuffer(int timeoutMs) {
566 synchronized (newFrameLock) {
567 if (renderedBuffer == null && timeoutMs > 0 && isWaitingForTexture()) {
568 try {
569 newFrameLock.wait(timeoutMs);
sakalb6760f92016-09-29 04:12:44 -0700570 } catch (InterruptedException e) {
Per488e75f2015-11-19 10:43:36 +0100571 // Restore the interrupted status by reinterrupting the thread.
572 Thread.currentThread().interrupt();
573 }
574 }
575 DecodedTextureBuffer returnedBuffer = renderedBuffer;
576 renderedBuffer = null;
577 return returnedBuffer;
578 }
579 }
580
581 public void release() {
nissea44e72c2016-05-27 00:27:59 -0700582 // SurfaceTextureHelper.stopListening() will block until any onTextureFrameAvailable() in
583 // progress is done. Therefore, the call must be outside any synchronized
Per488e75f2015-11-19 10:43:36 +0100584 // statement that is also used in the onTextureFrameAvailable() above to avoid deadlocks.
nissea44e72c2016-05-27 00:27:59 -0700585 surfaceTextureHelper.stopListening();
Per488e75f2015-11-19 10:43:36 +0100586 synchronized (newFrameLock) {
587 if (renderedBuffer != null) {
588 surfaceTextureHelper.returnTextureFrame();
589 renderedBuffer = null;
590 }
591 }
592 }
593 }
594
595 // Returns null if no decoded buffer is available, and otherwise a DecodedByteBuffer.
Magnus Jedvert7e319372015-10-02 15:49:38 +0200596 // Throws IllegalStateException if call is made on the wrong thread, if color format changes to an
597 // unsupported format, or if |mediaCodec| is not in the Executing state. Throws CodecException
598 // upon codec error.
Per488e75f2015-11-19 10:43:36 +0100599 private DecodedOutputBuffer dequeueOutputBuffer(int dequeueTimeoutMs) {
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000600 checkOnMediaCodecThread();
Per488e75f2015-11-19 10:43:36 +0100601 if (decodeStartTimeMs.isEmpty()) {
602 return null;
603 }
Magnus Jedvert7e319372015-10-02 15:49:38 +0200604 // Drain the decoder until receiving a decoded buffer or hitting
605 // MediaCodec.INFO_TRY_AGAIN_LATER.
606 final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
607 while (true) {
sakalb6760f92016-09-29 04:12:44 -0700608 final int result =
609 mediaCodec.dequeueOutputBuffer(info, TimeUnit.MILLISECONDS.toMicros(dequeueTimeoutMs));
Magnus Jedvert7e319372015-10-02 15:49:38 +0200610 switch (result) {
Magnus Jedvert7e319372015-10-02 15:49:38 +0200611 case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000612 outputBuffers = mediaCodec.getOutputBuffers();
Jiayang Liu5975b3c2015-09-16 13:40:53 -0700613 Logging.d(TAG, "Decoder output buffers changed: " + outputBuffers.length);
Per488e75f2015-11-19 10:43:36 +0100614 if (hasDecodedFirstFrame) {
615 throw new RuntimeException("Unexpected output buffer change event.");
616 }
Magnus Jedvert7e319372015-10-02 15:49:38 +0200617 break;
618 case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000619 MediaFormat format = mediaCodec.getOutputFormat();
Jiayang Liu5975b3c2015-09-16 13:40:53 -0700620 Logging.d(TAG, "Decoder format changed: " + format.toString());
magjed0c29de52017-03-02 00:55:32 -0800621 final int newWidth;
622 final int newHeight;
623 if (format.containsKey(FORMAT_KEY_CROP_LEFT) && format.containsKey(FORMAT_KEY_CROP_RIGHT)
624 && format.containsKey(FORMAT_KEY_CROP_BOTTOM)
625 && format.containsKey(FORMAT_KEY_CROP_TOP)) {
626 newWidth = 1 + format.getInteger(FORMAT_KEY_CROP_RIGHT)
627 - format.getInteger(FORMAT_KEY_CROP_LEFT);
628 newHeight = 1 + format.getInteger(FORMAT_KEY_CROP_BOTTOM)
629 - format.getInteger(FORMAT_KEY_CROP_TOP);
630 } else {
631 newWidth = format.getInteger(MediaFormat.KEY_WIDTH);
632 newHeight = format.getInteger(MediaFormat.KEY_HEIGHT);
Per488e75f2015-11-19 10:43:36 +0100633 }
magjed0c29de52017-03-02 00:55:32 -0800634 if (hasDecodedFirstFrame && (newWidth != width || newHeight != height)) {
635 throw new RuntimeException("Unexpected size change. Configured " + width + "*" + height
636 + ". New " + newWidth + "*" + newHeight);
637 }
638 width = newWidth;
639 height = newHeight;
Per488e75f2015-11-19 10:43:36 +0100640
glaznev@webrtc.org99678452014-09-15 17:52:42 +0000641 if (!useSurface && format.containsKey(MediaFormat.KEY_COLOR_FORMAT)) {
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000642 colorFormat = format.getInteger(MediaFormat.KEY_COLOR_FORMAT);
Jiayang Liu5975b3c2015-09-16 13:40:53 -0700643 Logging.d(TAG, "Color: 0x" + Integer.toHexString(colorFormat));
Magnus Jedvert7e319372015-10-02 15:49:38 +0200644 if (!supportedColorList.contains(colorFormat)) {
645 throw new IllegalStateException("Non supported color format: " + colorFormat);
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000646 }
647 }
magjed0c29de52017-03-02 00:55:32 -0800648 if (format.containsKey(FORMAT_KEY_STRIDE)) {
649 stride = format.getInteger(FORMAT_KEY_STRIDE);
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000650 }
magjed0c29de52017-03-02 00:55:32 -0800651 if (format.containsKey(FORMAT_KEY_SLICE_HEIGHT)) {
652 sliceHeight = format.getInteger(FORMAT_KEY_SLICE_HEIGHT);
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000653 }
Magnus Jedvert7e319372015-10-02 15:49:38 +0200654 Logging.d(TAG, "Frame stride and slice height: " + stride + " x " + sliceHeight);
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000655 stride = Math.max(width, stride);
656 sliceHeight = Math.max(height, sliceHeight);
Magnus Jedvert7e319372015-10-02 15:49:38 +0200657 break;
Per488e75f2015-11-19 10:43:36 +0100658 case MediaCodec.INFO_TRY_AGAIN_LATER:
659 return null;
Magnus Jedvert7e319372015-10-02 15:49:38 +0200660 default:
Per488e75f2015-11-19 10:43:36 +0100661 hasDecodedFirstFrame = true;
662 TimeStamps timeStamps = decodeStartTimeMs.remove();
magjedcedddbd2016-02-26 09:36:04 -0800663 long decodeTimeMs = SystemClock.elapsedRealtime() - timeStamps.decodeStartTimeMs;
664 if (decodeTimeMs > MAX_DECODE_TIME_MS) {
Alex Glaznevd5704842016-06-27 11:51:10 -0700665 Logging.e(TAG, "Very high decode time: " + decodeTimeMs + "ms"
sakalb6760f92016-09-29 04:12:44 -0700666 + ". Q size: " + decodeStartTimeMs.size()
667 + ". Might be caused by resuming H264 decoding after a pause.");
magjedcedddbd2016-02-26 09:36:04 -0800668 decodeTimeMs = MAX_DECODE_TIME_MS;
669 }
sakalb6760f92016-09-29 04:12:44 -0700670 return new DecodedOutputBuffer(result, info.offset, info.size,
671 TimeUnit.MICROSECONDS.toMillis(info.presentationTimeUs), timeStamps.timeStampMs,
672 timeStamps.ntpTimeStampMs, decodeTimeMs, SystemClock.elapsedRealtime());
673 }
perkj9cb89822015-11-11 03:27:01 -0800674 }
perkj9cb89822015-11-11 03:27:01 -0800675 }
676
Per488e75f2015-11-19 10:43:36 +0100677 // Returns null if no decoded buffer is available, and otherwise a DecodedTextureBuffer.
678 // Throws IllegalStateException if call is made on the wrong thread, if color format changes to an
679 // unsupported format, or if |mediaCodec| is not in the Executing state. Throws CodecException
680 // upon codec error. If |dequeueTimeoutMs| > 0, the oldest decoded frame will be dropped if
681 // a frame can't be returned.
682 private DecodedTextureBuffer dequeueTextureBuffer(int dequeueTimeoutMs) {
683 checkOnMediaCodecThread();
684 if (!useSurface) {
685 throw new IllegalStateException("dequeueTexture() called for byte buffer decoding.");
686 }
687 DecodedOutputBuffer outputBuffer = dequeueOutputBuffer(dequeueTimeoutMs);
688 if (outputBuffer != null) {
689 dequeuedSurfaceOutputBuffers.add(outputBuffer);
690 }
691
692 MaybeRenderDecodedTextureBuffer();
693 // Check if there is texture ready now by waiting max |dequeueTimeoutMs|.
694 DecodedTextureBuffer renderedBuffer = textureListener.dequeueTextureBuffer(dequeueTimeoutMs);
695 if (renderedBuffer != null) {
696 MaybeRenderDecodedTextureBuffer();
697 return renderedBuffer;
698 }
699
700 if ((dequeuedSurfaceOutputBuffers.size()
sakalb6760f92016-09-29 04:12:44 -0700701 >= Math.min(MAX_QUEUED_OUTPUTBUFFERS, outputBuffers.length)
702 || (dequeueTimeoutMs > 0 && !dequeuedSurfaceOutputBuffers.isEmpty()))) {
Per488e75f2015-11-19 10:43:36 +0100703 ++droppedFrames;
704 // Drop the oldest frame still in dequeuedSurfaceOutputBuffers.
705 // The oldest frame is owned by |textureListener| and can't be dropped since
706 // mediaCodec.releaseOutputBuffer has already been called.
707 final DecodedOutputBuffer droppedFrame = dequeuedSurfaceOutputBuffers.remove();
708 if (dequeueTimeoutMs > 0) {
perkj7baf79f2015-11-24 06:26:38 -0800709 // TODO(perkj): Re-add the below log when VideoRenderGUI has been removed or fixed to
710 // return the one and only texture even if it does not render.
glaznevae95ff32016-02-04 11:47:12 -0800711 Logging.w(TAG, "Draining decoder. Dropping frame with TS: "
sakalb6760f92016-09-29 04:12:44 -0700712 + droppedFrame.presentationTimeStampMs + ". Total number of dropped frames: "
713 + droppedFrames);
Per488e75f2015-11-19 10:43:36 +0100714 } else {
sakalb6760f92016-09-29 04:12:44 -0700715 Logging.w(TAG, "Too many output buffers " + dequeuedSurfaceOutputBuffers.size()
716 + ". Dropping frame with TS: " + droppedFrame.presentationTimeStampMs
717 + ". Total number of dropped frames: " + droppedFrames);
Per488e75f2015-11-19 10:43:36 +0100718 }
719
720 mediaCodec.releaseOutputBuffer(droppedFrame.index, false /* render */);
sakalb6760f92016-09-29 04:12:44 -0700721 return new DecodedTextureBuffer(0, null, droppedFrame.presentationTimeStampMs,
722 droppedFrame.timeStampMs, droppedFrame.ntpTimeStampMs, droppedFrame.decodeTimeMs,
Per488e75f2015-11-19 10:43:36 +0100723 SystemClock.elapsedRealtime() - droppedFrame.endDecodeTimeMs);
724 }
725 return null;
726 }
727
728 private void MaybeRenderDecodedTextureBuffer() {
729 if (dequeuedSurfaceOutputBuffers.isEmpty() || textureListener.isWaitingForTexture()) {
730 return;
731 }
732 // Get the first frame in the queue and render to the decoder output surface.
733 final DecodedOutputBuffer buffer = dequeuedSurfaceOutputBuffers.remove();
734 textureListener.addBufferToRender(buffer);
735 mediaCodec.releaseOutputBuffer(buffer.index, true /* render */);
736 }
737
magjed44bf6f52015-10-03 02:08:00 -0700738 // Release a dequeued output byte buffer back to the codec for re-use. Should only be called for
739 // non-surface decoding.
740 // Throws IllegalStateException if the call is made on the wrong thread, if codec is configured
741 // for surface decoding, or if |mediaCodec| is not in the Executing state. Throws
742 // MediaCodec.CodecException upon codec error.
Per488e75f2015-11-19 10:43:36 +0100743 private void returnDecodedOutputBuffer(int index)
Magnus Jedvert7e319372015-10-02 15:49:38 +0200744 throws IllegalStateException, MediaCodec.CodecException {
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000745 checkOnMediaCodecThread();
magjed44bf6f52015-10-03 02:08:00 -0700746 if (useSurface) {
Per488e75f2015-11-19 10:43:36 +0100747 throw new IllegalStateException("returnDecodedOutputBuffer() called for surface decoding.");
magjed44bf6f52015-10-03 02:08:00 -0700748 }
749 mediaCodec.releaseOutputBuffer(index, false /* render */);
glaznev@webrtc.orgefe4b9a2014-07-22 17:44:53 +0000750 }
751}