glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 1 | /* |
kjellander | b24317b | 2016-02-10 07:54:43 -0800 | [diff] [blame] | 2 | * Copyright 2014 The WebRTC project authors. All Rights Reserved. |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 3 | * |
kjellander | b24317b | 2016-02-10 07:54:43 -0800 | [diff] [blame] | 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 9 | */ |
| 10 | |
| 11 | package org.webrtc; |
| 12 | |
| 13 | import android.media.MediaCodec; |
buildbot@webrtc.org | a09a999 | 2014-08-13 17:26:08 +0000 | [diff] [blame] | 14 | import android.media.MediaCodecInfo; |
glaznev@webrtc.org | 9967845 | 2014-09-15 17:52:42 +0000 | [diff] [blame] | 15 | import android.media.MediaCodecInfo.CodecCapabilities; |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 16 | import android.media.MediaCodecList; |
| 17 | import android.media.MediaFormat; |
| 18 | import android.os.Build; |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 19 | import android.os.SystemClock; |
glaznev@webrtc.org | 9967845 | 2014-09-15 17:52:42 +0000 | [diff] [blame] | 20 | import android.view.Surface; |
| 21 | |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 22 | import java.nio.ByteBuffer; |
Magnus Jedvert | 7e31937 | 2015-10-02 15:49:38 +0200 | [diff] [blame] | 23 | import java.util.Arrays; |
Alex Glaznev | eee86a6 | 2016-01-29 14:17:07 -0800 | [diff] [blame] | 24 | import java.util.HashSet; |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 25 | import java.util.LinkedList; |
Magnus Jedvert | 91b348c | 2015-10-07 22:57:06 +0200 | [diff] [blame] | 26 | import java.util.List; |
Sami Kalliomaki | d3235f0 | 2016-08-02 15:44:04 +0200 | [diff] [blame] | 27 | import java.util.Queue; |
Alex Glaznev | eee86a6 | 2016-01-29 14:17:07 -0800 | [diff] [blame] | 28 | import java.util.Set; |
Alex Glaznev | 5c3da4b | 2015-10-30 15:31:07 -0700 | [diff] [blame] | 29 | import java.util.concurrent.CountDownLatch; |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 30 | import java.util.concurrent.TimeUnit; |
magjed | 8c425aa | 2015-10-22 16:52:39 -0700 | [diff] [blame] | 31 | |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 32 | // Java-side of peerconnection_jni.cc:MediaCodecVideoDecoder. |
| 33 | // This class is an implementation detail of the Java PeerConnection API. |
Patrik Höglund | 68876f9 | 2015-11-12 17:36:48 +0100 | [diff] [blame] | 34 | @SuppressWarnings("deprecation") |
Alex Glaznev | 908e77b | 2015-04-22 09:25:34 -0700 | [diff] [blame] | 35 | public class MediaCodecVideoDecoder { |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 36 | // This class is constructed, operated, and destroyed by its C++ incarnation, |
| 37 | // so the class and its methods have non-public visibility. The API this |
| 38 | // class exposes aims to mimic the webrtc::VideoDecoder API as closely as |
| 39 | // possibly to minimize the amount of translation work necessary. |
| 40 | |
| 41 | private static final String TAG = "MediaCodecVideoDecoder"; |
magjed | cedddbd | 2016-02-26 09:36:04 -0800 | [diff] [blame] | 42 | private static final long MAX_DECODE_TIME_MS = 200; |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 43 | |
magjed | 0c29de5 | 2017-03-02 00:55:32 -0800 | [diff] [blame] | 44 | // TODO(magjed): Use MediaFormat constants when part of the public API. |
| 45 | private static final String FORMAT_KEY_STRIDE = "stride"; |
| 46 | private static final String FORMAT_KEY_SLICE_HEIGHT = "slice-height"; |
| 47 | private static final String FORMAT_KEY_CROP_LEFT = "crop-left"; |
| 48 | private static final String FORMAT_KEY_CROP_RIGHT = "crop-right"; |
| 49 | private static final String FORMAT_KEY_CROP_TOP = "crop-top"; |
| 50 | private static final String FORMAT_KEY_CROP_BOTTOM = "crop-bottom"; |
| 51 | |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 52 | // Tracks webrtc::VideoCodecType. |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 53 | public enum VideoCodecType { VIDEO_CODEC_VP8, VIDEO_CODEC_VP9, VIDEO_CODEC_H264 } |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 54 | |
Alex Glaznev | eee86a6 | 2016-01-29 14:17:07 -0800 | [diff] [blame] | 55 | // Timeout for input buffer dequeue. |
| 56 | private static final int DEQUEUE_INPUT_TIMEOUT = 500000; |
| 57 | // Timeout for codec releasing. |
| 58 | private static final int MEDIA_CODEC_RELEASE_TIMEOUT_MS = 5000; |
| 59 | // Max number of output buffers queued before starting to drop decoded frames. |
| 60 | private static final int MAX_QUEUED_OUTPUTBUFFERS = 3; |
Alex Glaznev | c6aec4b | 2015-10-19 16:39:19 -0700 | [diff] [blame] | 61 | // Active running decoder instance. Set in initDecode() (called from native code) |
| 62 | // and reset to null in release() call. |
| 63 | private static MediaCodecVideoDecoder runningInstance = null; |
Alex Glaznev | 5c3da4b | 2015-10-30 15:31:07 -0700 | [diff] [blame] | 64 | private static MediaCodecVideoDecoderErrorCallback errorCallback = null; |
| 65 | private static int codecErrors = 0; |
Alex Glaznev | eee86a6 | 2016-01-29 14:17:07 -0800 | [diff] [blame] | 66 | // List of disabled codec types - can be set from application. |
| 67 | private static Set<String> hwDecoderDisabledTypes = new HashSet<String>(); |
Alex Glaznev | 5c3da4b | 2015-10-30 15:31:07 -0700 | [diff] [blame] | 68 | |
Alejandro Luebs | 69ddaef | 2015-10-09 15:46:09 -0700 | [diff] [blame] | 69 | private Thread mediaCodecThread; |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 70 | private MediaCodec mediaCodec; |
| 71 | private ByteBuffer[] inputBuffers; |
| 72 | private ByteBuffer[] outputBuffers; |
| 73 | private static final String VP8_MIME_TYPE = "video/x-vnd.on2.vp8"; |
Alex Glaznev | 69a7fd5 | 2015-11-10 10:25:40 -0800 | [diff] [blame] | 74 | private static final String VP9_MIME_TYPE = "video/x-vnd.on2.vp9"; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 75 | private static final String H264_MIME_TYPE = "video/avc"; |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 76 | // List of supported HW VP8 decoders. |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 77 | private static final String[] supportedVp8HwCodecPrefixes = { |
| 78 | "OMX.qcom.", "OMX.Nvidia.", "OMX.Exynos.", "OMX.Intel."}; |
Alex Glaznev | 69a7fd5 | 2015-11-10 10:25:40 -0800 | [diff] [blame] | 79 | // List of supported HW VP9 decoders. |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 80 | private static final String[] supportedVp9HwCodecPrefixes = {"OMX.qcom.", "OMX.Exynos."}; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 81 | // List of supported HW H.264 decoders. |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 82 | private static final String[] supportedH264HwCodecPrefixes = { |
| 83 | "OMX.qcom.", "OMX.Intel.", "OMX.Exynos."}; |
glaznev | 0c1d060 | 2017-01-27 12:24:24 -0800 | [diff] [blame] | 84 | // List of supported HW H.264 high profile decoders. |
glaznev | cca0f6c | 2017-06-14 10:20:54 -0700 | [diff] [blame] | 85 | private static final String supportedQcomH264HighProfileHwCodecPrefix = "OMX.qcom."; |
| 86 | private static final String supportedExynosH264HighProfileHwCodecPrefix = "OMX.Exynos."; |
glaznev | 893a7ee | 2016-09-22 10:44:30 -0700 | [diff] [blame] | 87 | |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 88 | // NV12 color format supported by QCOM codec, but not declared in MediaCodec - |
| 89 | // see /hardware/qcom/media/mm-core/inc/OMX_QCOMExtns.h |
glaznev | 893a7ee | 2016-09-22 10:44:30 -0700 | [diff] [blame] | 90 | private static final int COLOR_QCOM_FORMATYVU420PackedSemiPlanar32m4ka = 0x7FA30C01; |
| 91 | private static final int COLOR_QCOM_FORMATYVU420PackedSemiPlanar16m4ka = 0x7FA30C02; |
| 92 | private static final int COLOR_QCOM_FORMATYVU420PackedSemiPlanar64x32Tile2m8ka = 0x7FA30C03; |
| 93 | private static final int COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m = 0x7FA30C04; |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 94 | // Allowable color formats supported by codec - in order of preference. |
Magnus Jedvert | 7e31937 | 2015-10-02 15:49:38 +0200 | [diff] [blame] | 95 | private static final List<Integer> supportedColorList = Arrays.asList( |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 96 | CodecCapabilities.COLOR_FormatYUV420Planar, CodecCapabilities.COLOR_FormatYUV420SemiPlanar, |
| 97 | CodecCapabilities.COLOR_QCOM_FormatYUV420SemiPlanar, |
| 98 | COLOR_QCOM_FORMATYVU420PackedSemiPlanar32m4ka, COLOR_QCOM_FORMATYVU420PackedSemiPlanar16m4ka, |
| 99 | COLOR_QCOM_FORMATYVU420PackedSemiPlanar64x32Tile2m8ka, |
| 100 | COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m); |
glaznev | 893a7ee | 2016-09-22 10:44:30 -0700 | [diff] [blame] | 101 | |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 102 | private int colorFormat; |
| 103 | private int width; |
| 104 | private int height; |
| 105 | private int stride; |
| 106 | private int sliceHeight; |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 107 | private boolean hasDecodedFirstFrame; |
| 108 | private final Queue<TimeStamps> decodeStartTimeMs = new LinkedList<TimeStamps>(); |
glaznev@webrtc.org | 9967845 | 2014-09-15 17:52:42 +0000 | [diff] [blame] | 109 | private boolean useSurface; |
Per | c01c254 | 2015-11-13 16:58:26 +0100 | [diff] [blame] | 110 | |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 111 | // The below variables are only used when decoding to a Surface. |
| 112 | private TextureListener textureListener; |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 113 | private int droppedFrames; |
| 114 | private Surface surface = null; |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 115 | private final Queue<DecodedOutputBuffer> dequeuedSurfaceOutputBuffers = |
| 116 | new LinkedList<DecodedOutputBuffer>(); |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 117 | |
Alex Glaznev | 5c3da4b | 2015-10-30 15:31:07 -0700 | [diff] [blame] | 118 | // MediaCodec error handler - invoked when critical error happens which may prevent |
| 119 | // further use of media codec API. Now it means that one of media codec instances |
| 120 | // is hanging and can no longer be used in the next call. |
| 121 | public static interface MediaCodecVideoDecoderErrorCallback { |
| 122 | void onMediaCodecVideoDecoderCriticalError(int codecErrors); |
| 123 | } |
| 124 | |
| 125 | public static void setErrorCallback(MediaCodecVideoDecoderErrorCallback errorCallback) { |
| 126 | Logging.d(TAG, "Set error callback"); |
| 127 | MediaCodecVideoDecoder.errorCallback = errorCallback; |
| 128 | } |
| 129 | |
Alex Glaznev | eee86a6 | 2016-01-29 14:17:07 -0800 | [diff] [blame] | 130 | // Functions to disable HW decoding - can be called from applications for platforms |
| 131 | // which have known HW decoding problems. |
| 132 | public static void disableVp8HwCodec() { |
| 133 | Logging.w(TAG, "VP8 decoding is disabled by application."); |
| 134 | hwDecoderDisabledTypes.add(VP8_MIME_TYPE); |
| 135 | } |
| 136 | |
| 137 | public static void disableVp9HwCodec() { |
| 138 | Logging.w(TAG, "VP9 decoding is disabled by application."); |
| 139 | hwDecoderDisabledTypes.add(VP9_MIME_TYPE); |
| 140 | } |
| 141 | |
| 142 | public static void disableH264HwCodec() { |
| 143 | Logging.w(TAG, "H.264 decoding is disabled by application."); |
| 144 | hwDecoderDisabledTypes.add(H264_MIME_TYPE); |
| 145 | } |
| 146 | |
| 147 | // Functions to query if HW decoding is supported. |
| 148 | public static boolean isVp8HwSupported() { |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 149 | return !hwDecoderDisabledTypes.contains(VP8_MIME_TYPE) |
| 150 | && (findDecoder(VP8_MIME_TYPE, supportedVp8HwCodecPrefixes) != null); |
Alex Glaznev | eee86a6 | 2016-01-29 14:17:07 -0800 | [diff] [blame] | 151 | } |
| 152 | |
| 153 | public static boolean isVp9HwSupported() { |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 154 | return !hwDecoderDisabledTypes.contains(VP9_MIME_TYPE) |
| 155 | && (findDecoder(VP9_MIME_TYPE, supportedVp9HwCodecPrefixes) != null); |
Alex Glaznev | eee86a6 | 2016-01-29 14:17:07 -0800 | [diff] [blame] | 156 | } |
| 157 | |
| 158 | public static boolean isH264HwSupported() { |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 159 | return !hwDecoderDisabledTypes.contains(H264_MIME_TYPE) |
| 160 | && (findDecoder(H264_MIME_TYPE, supportedH264HwCodecPrefixes) != null); |
Alex Glaznev | eee86a6 | 2016-01-29 14:17:07 -0800 | [diff] [blame] | 161 | } |
| 162 | |
glaznev | 0c1d060 | 2017-01-27 12:24:24 -0800 | [diff] [blame] | 163 | public static boolean isH264HighProfileHwSupported() { |
glaznev | cca0f6c | 2017-06-14 10:20:54 -0700 | [diff] [blame] | 164 | if (hwDecoderDisabledTypes.contains(H264_MIME_TYPE)) { |
| 165 | return false; |
| 166 | } |
| 167 | // Support H.264 HP decoding on QCOM chips for Android L and above. |
| 168 | if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP |
| 169 | && findDecoder(H264_MIME_TYPE, new String[] {supportedQcomH264HighProfileHwCodecPrefix}) |
| 170 | != null) { |
| 171 | return true; |
| 172 | } |
| 173 | // Support H.264 HP decoding on Exynos chips for Android M and above. |
| 174 | if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M |
| 175 | && findDecoder(H264_MIME_TYPE, new String[] {supportedExynosH264HighProfileHwCodecPrefix}) |
| 176 | != null) { |
| 177 | return true; |
| 178 | } |
| 179 | return false; |
glaznev | 0c1d060 | 2017-01-27 12:24:24 -0800 | [diff] [blame] | 180 | } |
| 181 | |
Alex Glaznev | eee86a6 | 2016-01-29 14:17:07 -0800 | [diff] [blame] | 182 | public static void printStackTrace() { |
| 183 | if (runningInstance != null && runningInstance.mediaCodecThread != null) { |
| 184 | StackTraceElement[] mediaCodecStackTraces = runningInstance.mediaCodecThread.getStackTrace(); |
| 185 | if (mediaCodecStackTraces.length > 0) { |
| 186 | Logging.d(TAG, "MediaCodecVideoDecoder stacks trace:"); |
| 187 | for (StackTraceElement stackTrace : mediaCodecStackTraces) { |
| 188 | Logging.d(TAG, stackTrace.toString()); |
| 189 | } |
| 190 | } |
| 191 | } |
| 192 | } |
| 193 | |
| 194 | // Helper struct for findDecoder() below. |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 195 | private static class DecoderProperties { |
glaznev@webrtc.org | 9967845 | 2014-09-15 17:52:42 +0000 | [diff] [blame] | 196 | public DecoderProperties(String codecName, int colorFormat) { |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 197 | this.codecName = codecName; |
| 198 | this.colorFormat = colorFormat; |
| 199 | } |
| 200 | public final String codecName; // OpenMax component name for VP8 codec. |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 201 | public final int colorFormat; // Color format supported by codec. |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 202 | } |
| 203 | |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 204 | private static DecoderProperties findDecoder(String mime, String[] supportedCodecPrefixes) { |
glaznev@webrtc.org | 25cc745 | 2014-10-02 16:58:05 +0000 | [diff] [blame] | 205 | if (Build.VERSION.SDK_INT < Build.VERSION_CODES.KITKAT) { |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 206 | return null; // MediaCodec.setParameters is missing. |
glaznev@webrtc.org | 25cc745 | 2014-10-02 16:58:05 +0000 | [diff] [blame] | 207 | } |
Alex Glaznev | 69a7fd5 | 2015-11-10 10:25:40 -0800 | [diff] [blame] | 208 | Logging.d(TAG, "Trying to find HW decoder for mime " + mime); |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 209 | for (int i = 0; i < MediaCodecList.getCodecCount(); ++i) { |
Alex Glaznev | 0060c56 | 2016-08-08 12:27:24 -0700 | [diff] [blame] | 210 | MediaCodecInfo info = null; |
| 211 | try { |
| 212 | info = MediaCodecList.getCodecInfoAt(i); |
| 213 | } catch (IllegalArgumentException e) { |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 214 | Logging.e(TAG, "Cannot retrieve decoder codec info", e); |
Alex Glaznev | 0060c56 | 2016-08-08 12:27:24 -0700 | [diff] [blame] | 215 | } |
| 216 | if (info == null || info.isEncoder()) { |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 217 | continue; |
| 218 | } |
| 219 | String name = null; |
| 220 | for (String mimeType : info.getSupportedTypes()) { |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 221 | if (mimeType.equals(mime)) { |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 222 | name = info.getName(); |
| 223 | break; |
| 224 | } |
| 225 | } |
| 226 | if (name == null) { |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 227 | continue; // No HW support in this codec; try the next one. |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 228 | } |
Alex Glaznev | 69a7fd5 | 2015-11-10 10:25:40 -0800 | [diff] [blame] | 229 | Logging.d(TAG, "Found candidate decoder " + name); |
glaznev@webrtc.org | 9967845 | 2014-09-15 17:52:42 +0000 | [diff] [blame] | 230 | |
glaznev@webrtc.org | 25cc745 | 2014-10-02 16:58:05 +0000 | [diff] [blame] | 231 | // Check if this is supported decoder. |
glaznev@webrtc.org | 9967845 | 2014-09-15 17:52:42 +0000 | [diff] [blame] | 232 | boolean supportedCodec = false; |
glaznev@webrtc.org | 25cc745 | 2014-10-02 16:58:05 +0000 | [diff] [blame] | 233 | for (String codecPrefix : supportedCodecPrefixes) { |
| 234 | if (name.startsWith(codecPrefix)) { |
glaznev@webrtc.org | 9967845 | 2014-09-15 17:52:42 +0000 | [diff] [blame] | 235 | supportedCodec = true; |
| 236 | break; |
| 237 | } |
| 238 | } |
| 239 | if (!supportedCodec) { |
| 240 | continue; |
| 241 | } |
| 242 | |
| 243 | // Check if codec supports either yuv420 or nv12. |
Alex Glaznev | 0060c56 | 2016-08-08 12:27:24 -0700 | [diff] [blame] | 244 | CodecCapabilities capabilities; |
| 245 | try { |
| 246 | capabilities = info.getCapabilitiesForType(mime); |
| 247 | } catch (IllegalArgumentException e) { |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 248 | Logging.e(TAG, "Cannot retrieve decoder capabilities", e); |
Alex Glaznev | 0060c56 | 2016-08-08 12:27:24 -0700 | [diff] [blame] | 249 | continue; |
| 250 | } |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 251 | for (int colorFormat : capabilities.colorFormats) { |
Jiayang Liu | 5975b3c | 2015-09-16 13:40:53 -0700 | [diff] [blame] | 252 | Logging.v(TAG, " Color: 0x" + Integer.toHexString(colorFormat)); |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 253 | } |
glaznev@webrtc.org | 9967845 | 2014-09-15 17:52:42 +0000 | [diff] [blame] | 254 | for (int supportedColorFormat : supportedColorList) { |
| 255 | for (int codecColorFormat : capabilities.colorFormats) { |
| 256 | if (codecColorFormat == supportedColorFormat) { |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 257 | // Found supported HW decoder. |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 258 | Logging.d(TAG, "Found target decoder " + name + ". Color: 0x" |
| 259 | + Integer.toHexString(codecColorFormat)); |
glaznev@webrtc.org | 9967845 | 2014-09-15 17:52:42 +0000 | [diff] [blame] | 260 | return new DecoderProperties(name, codecColorFormat); |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 261 | } |
| 262 | } |
| 263 | } |
| 264 | } |
Alex Glaznev | 69a7fd5 | 2015-11-10 10:25:40 -0800 | [diff] [blame] | 265 | Logging.d(TAG, "No HW decoder found for mime " + mime); |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 266 | return null; // No HW decoder. |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 267 | } |
| 268 | |
Magnus Jedvert | 7e31937 | 2015-10-02 15:49:38 +0200 | [diff] [blame] | 269 | private void checkOnMediaCodecThread() throws IllegalStateException { |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 270 | if (mediaCodecThread.getId() != Thread.currentThread().getId()) { |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 271 | throw new IllegalStateException("MediaCodecVideoDecoder previously operated on " |
| 272 | + mediaCodecThread + " but is now called on " + Thread.currentThread()); |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 273 | } |
| 274 | } |
| 275 | |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 276 | // Pass null in |surfaceTextureHelper| to configure the codec for ByteBuffer output. |
| 277 | private boolean initDecode( |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 278 | VideoCodecType type, int width, int height, SurfaceTextureHelper surfaceTextureHelper) { |
Alejandro Luebs | 69ddaef | 2015-10-09 15:46:09 -0700 | [diff] [blame] | 279 | if (mediaCodecThread != null) { |
Alex Glaznev | 6a4a03c | 2016-03-04 14:10:50 -0800 | [diff] [blame] | 280 | throw new RuntimeException("initDecode: Forgot to release()?"); |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 281 | } |
Alex Glaznev | 6a4a03c | 2016-03-04 14:10:50 -0800 | [diff] [blame] | 282 | |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 283 | String mime = null; |
Alex Glaznev | 6a4a03c | 2016-03-04 14:10:50 -0800 | [diff] [blame] | 284 | useSurface = (surfaceTextureHelper != null); |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 285 | String[] supportedCodecPrefixes = null; |
| 286 | if (type == VideoCodecType.VIDEO_CODEC_VP8) { |
| 287 | mime = VP8_MIME_TYPE; |
| 288 | supportedCodecPrefixes = supportedVp8HwCodecPrefixes; |
Alex Glaznev | 69a7fd5 | 2015-11-10 10:25:40 -0800 | [diff] [blame] | 289 | } else if (type == VideoCodecType.VIDEO_CODEC_VP9) { |
| 290 | mime = VP9_MIME_TYPE; |
| 291 | supportedCodecPrefixes = supportedVp9HwCodecPrefixes; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 292 | } else if (type == VideoCodecType.VIDEO_CODEC_H264) { |
| 293 | mime = H264_MIME_TYPE; |
| 294 | supportedCodecPrefixes = supportedH264HwCodecPrefixes; |
| 295 | } else { |
Alex Glaznev | 6a4a03c | 2016-03-04 14:10:50 -0800 | [diff] [blame] | 296 | throw new RuntimeException("initDecode: Non-supported codec " + type); |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 297 | } |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 298 | DecoderProperties properties = findDecoder(mime, supportedCodecPrefixes); |
| 299 | if (properties == null) { |
| 300 | throw new RuntimeException("Cannot find HW decoder for " + type); |
| 301 | } |
Alex Glaznev | 6a4a03c | 2016-03-04 14:10:50 -0800 | [diff] [blame] | 302 | |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 303 | Logging.d(TAG, "Java initDecode: " + type + " : " + width + " x " + height + ". Color: 0x" |
| 304 | + Integer.toHexString(properties.colorFormat) + ". Use Surface: " + useSurface); |
Alex Glaznev | 6a4a03c | 2016-03-04 14:10:50 -0800 | [diff] [blame] | 305 | |
Alex Glaznev | c6aec4b | 2015-10-19 16:39:19 -0700 | [diff] [blame] | 306 | runningInstance = this; // Decoder is now running and can be queried for stack traces. |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 307 | mediaCodecThread = Thread.currentThread(); |
| 308 | try { |
| 309 | this.width = width; |
| 310 | this.height = height; |
| 311 | stride = width; |
| 312 | sliceHeight = height; |
glaznev@webrtc.org | 9967845 | 2014-09-15 17:52:42 +0000 | [diff] [blame] | 313 | |
| 314 | if (useSurface) { |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 315 | textureListener = new TextureListener(surfaceTextureHelper); |
| 316 | surface = new Surface(surfaceTextureHelper.getSurfaceTexture()); |
Magnus Jedvert | 80cf97c | 2015-06-11 10:08:59 +0200 | [diff] [blame] | 317 | } |
glaznev@webrtc.org | 9967845 | 2014-09-15 17:52:42 +0000 | [diff] [blame] | 318 | |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 319 | MediaFormat format = MediaFormat.createVideoFormat(mime, width, height); |
glaznev@webrtc.org | 9967845 | 2014-09-15 17:52:42 +0000 | [diff] [blame] | 320 | if (!useSurface) { |
| 321 | format.setInteger(MediaFormat.KEY_COLOR_FORMAT, properties.colorFormat); |
| 322 | } |
Jiayang Liu | 5975b3c | 2015-09-16 13:40:53 -0700 | [diff] [blame] | 323 | Logging.d(TAG, " Format: " + format); |
Alex Glaznev | 6a4a03c | 2016-03-04 14:10:50 -0800 | [diff] [blame] | 324 | mediaCodec = MediaCodecVideoEncoder.createByCodecName(properties.codecName); |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 325 | if (mediaCodec == null) { |
Alex Glaznev | 325d414 | 2015-10-12 14:56:02 -0700 | [diff] [blame] | 326 | Logging.e(TAG, "Can not create media decoder"); |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 327 | return false; |
| 328 | } |
Magnus Jedvert | 7e31937 | 2015-10-02 15:49:38 +0200 | [diff] [blame] | 329 | mediaCodec.configure(format, surface, null, 0); |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 330 | mediaCodec.start(); |
Alex Glaznev | 6a4a03c | 2016-03-04 14:10:50 -0800 | [diff] [blame] | 331 | |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 332 | colorFormat = properties.colorFormat; |
| 333 | outputBuffers = mediaCodec.getOutputBuffers(); |
| 334 | inputBuffers = mediaCodec.getInputBuffers(); |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 335 | decodeStartTimeMs.clear(); |
| 336 | hasDecodedFirstFrame = false; |
| 337 | dequeuedSurfaceOutputBuffers.clear(); |
| 338 | droppedFrames = 0; |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 339 | Logging.d(TAG, |
| 340 | "Input buffers: " + inputBuffers.length + ". Output buffers: " + outputBuffers.length); |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 341 | return true; |
| 342 | } catch (IllegalStateException e) { |
Jiayang Liu | 5975b3c | 2015-09-16 13:40:53 -0700 | [diff] [blame] | 343 | Logging.e(TAG, "initDecode failed", e); |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 344 | return false; |
| 345 | } |
| 346 | } |
| 347 | |
Alex Glaznev | 6a4a03c | 2016-03-04 14:10:50 -0800 | [diff] [blame] | 348 | // Resets the decoder so it can start decoding frames with new resolution. |
| 349 | // Flushes MediaCodec and clears decoder output buffers. |
| 350 | private void reset(int width, int height) { |
| 351 | if (mediaCodecThread == null || mediaCodec == null) { |
| 352 | throw new RuntimeException("Incorrect reset call for non-initialized decoder."); |
| 353 | } |
| 354 | Logging.d(TAG, "Java reset: " + width + " x " + height); |
| 355 | |
| 356 | mediaCodec.flush(); |
| 357 | |
| 358 | this.width = width; |
| 359 | this.height = height; |
| 360 | decodeStartTimeMs.clear(); |
| 361 | dequeuedSurfaceOutputBuffers.clear(); |
| 362 | hasDecodedFirstFrame = false; |
| 363 | droppedFrames = 0; |
| 364 | } |
| 365 | |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 366 | private void release() { |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 367 | Logging.d(TAG, "Java releaseDecoder. Total number of dropped frames: " + droppedFrames); |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 368 | checkOnMediaCodecThread(); |
Alex Glaznev | 5c3da4b | 2015-10-30 15:31:07 -0700 | [diff] [blame] | 369 | |
| 370 | // Run Mediacodec stop() and release() on separate thread since sometime |
| 371 | // Mediacodec.stop() may hang. |
| 372 | final CountDownLatch releaseDone = new CountDownLatch(1); |
| 373 | |
| 374 | Runnable runMediaCodecRelease = new Runnable() { |
| 375 | @Override |
| 376 | public void run() { |
| 377 | try { |
| 378 | Logging.d(TAG, "Java releaseDecoder on release thread"); |
| 379 | mediaCodec.stop(); |
| 380 | mediaCodec.release(); |
| 381 | Logging.d(TAG, "Java releaseDecoder on release thread done"); |
| 382 | } catch (Exception e) { |
| 383 | Logging.e(TAG, "Media decoder release failed", e); |
| 384 | } |
| 385 | releaseDone.countDown(); |
| 386 | } |
| 387 | }; |
| 388 | new Thread(runMediaCodecRelease).start(); |
| 389 | |
| 390 | if (!ThreadUtils.awaitUninterruptibly(releaseDone, MEDIA_CODEC_RELEASE_TIMEOUT_MS)) { |
| 391 | Logging.e(TAG, "Media decoder release timeout"); |
| 392 | codecErrors++; |
| 393 | if (errorCallback != null) { |
| 394 | Logging.e(TAG, "Invoke codec error callback. Errors: " + codecErrors); |
| 395 | errorCallback.onMediaCodecVideoDecoderCriticalError(codecErrors); |
| 396 | } |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 397 | } |
Alex Glaznev | 5c3da4b | 2015-10-30 15:31:07 -0700 | [diff] [blame] | 398 | |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 399 | mediaCodec = null; |
| 400 | mediaCodecThread = null; |
Alex Glaznev | c6aec4b | 2015-10-19 16:39:19 -0700 | [diff] [blame] | 401 | runningInstance = null; |
glaznev@webrtc.org | 9967845 | 2014-09-15 17:52:42 +0000 | [diff] [blame] | 402 | if (useSurface) { |
| 403 | surface.release(); |
Magnus Jedvert | 7e31937 | 2015-10-02 15:49:38 +0200 | [diff] [blame] | 404 | surface = null; |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 405 | textureListener.release(); |
glaznev@webrtc.org | 9967845 | 2014-09-15 17:52:42 +0000 | [diff] [blame] | 406 | } |
Alex Glaznev | 325d414 | 2015-10-12 14:56:02 -0700 | [diff] [blame] | 407 | Logging.d(TAG, "Java releaseDecoder done"); |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 408 | } |
| 409 | |
| 410 | // Dequeue an input buffer and return its index, -1 if no input buffer is |
| 411 | // available, or -2 if the codec is no longer operative. |
| 412 | private int dequeueInputBuffer() { |
| 413 | checkOnMediaCodecThread(); |
| 414 | try { |
glaznev@webrtc.org | 9967845 | 2014-09-15 17:52:42 +0000 | [diff] [blame] | 415 | return mediaCodec.dequeueInputBuffer(DEQUEUE_INPUT_TIMEOUT); |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 416 | } catch (IllegalStateException e) { |
Jiayang Liu | 5975b3c | 2015-09-16 13:40:53 -0700 | [diff] [blame] | 417 | Logging.e(TAG, "dequeueIntputBuffer failed", e); |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 418 | return -2; |
| 419 | } |
| 420 | } |
| 421 | |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 422 | private boolean queueInputBuffer(int inputBufferIndex, int size, long presentationTimeStamUs, |
| 423 | long timeStampMs, long ntpTimeStamp) { |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 424 | checkOnMediaCodecThread(); |
| 425 | try { |
| 426 | inputBuffers[inputBufferIndex].position(0); |
| 427 | inputBuffers[inputBufferIndex].limit(size); |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 428 | decodeStartTimeMs.add( |
| 429 | new TimeStamps(SystemClock.elapsedRealtime(), timeStampMs, ntpTimeStamp)); |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 430 | mediaCodec.queueInputBuffer(inputBufferIndex, 0, size, presentationTimeStamUs, 0); |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 431 | return true; |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 432 | } catch (IllegalStateException e) { |
Jiayang Liu | 5975b3c | 2015-09-16 13:40:53 -0700 | [diff] [blame] | 433 | Logging.e(TAG, "decode failed", e); |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 434 | return false; |
| 435 | } |
| 436 | } |
| 437 | |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 438 | private static class TimeStamps { |
| 439 | public TimeStamps(long decodeStartTimeMs, long timeStampMs, long ntpTimeStampMs) { |
| 440 | this.decodeStartTimeMs = decodeStartTimeMs; |
| 441 | this.timeStampMs = timeStampMs; |
| 442 | this.ntpTimeStampMs = ntpTimeStampMs; |
| 443 | } |
Alex Glaznev | eee86a6 | 2016-01-29 14:17:07 -0800 | [diff] [blame] | 444 | // Time when this frame was queued for decoding. |
| 445 | private final long decodeStartTimeMs; |
| 446 | // Only used for bookkeeping in Java. Stores C++ inputImage._timeStamp value for input frame. |
| 447 | private final long timeStampMs; |
| 448 | // Only used for bookkeeping in Java. Stores C++ inputImage.ntp_time_ms_ value for input frame. |
| 449 | private final long ntpTimeStampMs; |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 450 | } |
| 451 | |
| 452 | // Helper struct for dequeueOutputBuffer() below. |
| 453 | private static class DecodedOutputBuffer { |
glaznev | 9429148 | 2016-02-01 13:17:18 -0800 | [diff] [blame] | 454 | public DecodedOutputBuffer(int index, int offset, int size, long presentationTimeStampMs, |
| 455 | long timeStampMs, long ntpTimeStampMs, long decodeTime, long endDecodeTime) { |
glaznev@webrtc.org | 9967845 | 2014-09-15 17:52:42 +0000 | [diff] [blame] | 456 | this.index = index; |
| 457 | this.offset = offset; |
| 458 | this.size = size; |
glaznev | 9429148 | 2016-02-01 13:17:18 -0800 | [diff] [blame] | 459 | this.presentationTimeStampMs = presentationTimeStampMs; |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 460 | this.timeStampMs = timeStampMs; |
| 461 | this.ntpTimeStampMs = ntpTimeStampMs; |
| 462 | this.decodeTimeMs = decodeTime; |
| 463 | this.endDecodeTimeMs = endDecodeTime; |
glaznev@webrtc.org | 9967845 | 2014-09-15 17:52:42 +0000 | [diff] [blame] | 464 | } |
| 465 | |
| 466 | private final int index; |
| 467 | private final int offset; |
| 468 | private final int size; |
glaznev | 9429148 | 2016-02-01 13:17:18 -0800 | [diff] [blame] | 469 | // Presentation timestamp returned in dequeueOutputBuffer call. |
| 470 | private final long presentationTimeStampMs; |
Alex Glaznev | eee86a6 | 2016-01-29 14:17:07 -0800 | [diff] [blame] | 471 | // C++ inputImage._timeStamp value for output frame. |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 472 | private final long timeStampMs; |
Alex Glaznev | eee86a6 | 2016-01-29 14:17:07 -0800 | [diff] [blame] | 473 | // C++ inputImage.ntp_time_ms_ value for output frame. |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 474 | private final long ntpTimeStampMs; |
| 475 | // Number of ms it took to decode this frame. |
| 476 | private final long decodeTimeMs; |
Alex Glaznev | eee86a6 | 2016-01-29 14:17:07 -0800 | [diff] [blame] | 477 | // System time when this frame decoding finished. |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 478 | private final long endDecodeTimeMs; |
glaznev@webrtc.org | 9967845 | 2014-09-15 17:52:42 +0000 | [diff] [blame] | 479 | } |
| 480 | |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 481 | // Helper struct for dequeueTextureBuffer() below. |
magjed | 44bf6f5 | 2015-10-03 02:08:00 -0700 | [diff] [blame] | 482 | private static class DecodedTextureBuffer { |
| 483 | private final int textureID; |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 484 | private final float[] transformMatrix; |
glaznev | 9429148 | 2016-02-01 13:17:18 -0800 | [diff] [blame] | 485 | // Presentation timestamp returned in dequeueOutputBuffer call. |
| 486 | private final long presentationTimeStampMs; |
Alex Glaznev | eee86a6 | 2016-01-29 14:17:07 -0800 | [diff] [blame] | 487 | // C++ inputImage._timeStamp value for output frame. |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 488 | private final long timeStampMs; |
Alex Glaznev | eee86a6 | 2016-01-29 14:17:07 -0800 | [diff] [blame] | 489 | // C++ inputImage.ntp_time_ms_ value for output frame. |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 490 | private final long ntpTimeStampMs; |
Alex Glaznev | eee86a6 | 2016-01-29 14:17:07 -0800 | [diff] [blame] | 491 | // Number of ms it took to decode this frame. |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 492 | private final long decodeTimeMs; |
| 493 | // Interval from when the frame finished decoding until this buffer has been created. |
| 494 | // Since there is only one texture, this interval depend on the time from when |
| 495 | // a frame is decoded and provided to C++ and until that frame is returned to the MediaCodec |
| 496 | // so that the texture can be updated with the next decoded frame. |
| 497 | private final long frameDelayMs; |
magjed | 44bf6f5 | 2015-10-03 02:08:00 -0700 | [diff] [blame] | 498 | |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 499 | // A DecodedTextureBuffer with zero |textureID| has special meaning and represents a frame |
| 500 | // that was dropped. |
glaznev | 9429148 | 2016-02-01 13:17:18 -0800 | [diff] [blame] | 501 | public DecodedTextureBuffer(int textureID, float[] transformMatrix, |
| 502 | long presentationTimeStampMs, long timeStampMs, long ntpTimeStampMs, long decodeTimeMs, |
| 503 | long frameDelay) { |
magjed | 44bf6f5 | 2015-10-03 02:08:00 -0700 | [diff] [blame] | 504 | this.textureID = textureID; |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 505 | this.transformMatrix = transformMatrix; |
glaznev | 9429148 | 2016-02-01 13:17:18 -0800 | [diff] [blame] | 506 | this.presentationTimeStampMs = presentationTimeStampMs; |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 507 | this.timeStampMs = timeStampMs; |
| 508 | this.ntpTimeStampMs = ntpTimeStampMs; |
| 509 | this.decodeTimeMs = decodeTimeMs; |
| 510 | this.frameDelayMs = frameDelay; |
magjed | 44bf6f5 | 2015-10-03 02:08:00 -0700 | [diff] [blame] | 511 | } |
| 512 | } |
| 513 | |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 514 | // Poll based texture listener. |
| 515 | private static class TextureListener |
| 516 | implements SurfaceTextureHelper.OnTextureFrameAvailableListener { |
| 517 | private final SurfaceTextureHelper surfaceTextureHelper; |
| 518 | // |newFrameLock| is used to synchronize arrival of new frames with wait()/notifyAll(). |
| 519 | private final Object newFrameLock = new Object(); |
| 520 | // |bufferToRender| is non-null when waiting for transition between addBufferToRender() to |
| 521 | // onTextureFrameAvailable(). |
| 522 | private DecodedOutputBuffer bufferToRender; |
| 523 | private DecodedTextureBuffer renderedBuffer; |
| 524 | |
| 525 | public TextureListener(SurfaceTextureHelper surfaceTextureHelper) { |
| 526 | this.surfaceTextureHelper = surfaceTextureHelper; |
magjed | 81e8e37 | 2016-03-03 02:11:44 -0800 | [diff] [blame] | 527 | surfaceTextureHelper.startListening(this); |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 528 | } |
| 529 | |
| 530 | public void addBufferToRender(DecodedOutputBuffer buffer) { |
| 531 | if (bufferToRender != null) { |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 532 | Logging.e(TAG, "Unexpected addBufferToRender() called while waiting for a texture."); |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 533 | throw new IllegalStateException("Waiting for a texture."); |
| 534 | } |
| 535 | bufferToRender = buffer; |
| 536 | } |
| 537 | |
| 538 | public boolean isWaitingForTexture() { |
| 539 | synchronized (newFrameLock) { |
| 540 | return bufferToRender != null; |
| 541 | } |
| 542 | } |
| 543 | |
| 544 | // Callback from |surfaceTextureHelper|. May be called on an arbitrary thread. |
| 545 | @Override |
| 546 | public void onTextureFrameAvailable( |
| 547 | int oesTextureId, float[] transformMatrix, long timestampNs) { |
| 548 | synchronized (newFrameLock) { |
| 549 | if (renderedBuffer != null) { |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 550 | Logging.e( |
| 551 | TAG, "Unexpected onTextureFrameAvailable() called while already holding a texture."); |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 552 | throw new IllegalStateException("Already holding a texture."); |
| 553 | } |
| 554 | // |timestampNs| is always zero on some Android versions. |
| 555 | renderedBuffer = new DecodedTextureBuffer(oesTextureId, transformMatrix, |
glaznev | 9429148 | 2016-02-01 13:17:18 -0800 | [diff] [blame] | 556 | bufferToRender.presentationTimeStampMs, bufferToRender.timeStampMs, |
| 557 | bufferToRender.ntpTimeStampMs, bufferToRender.decodeTimeMs, |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 558 | SystemClock.elapsedRealtime() - bufferToRender.endDecodeTimeMs); |
| 559 | bufferToRender = null; |
| 560 | newFrameLock.notifyAll(); |
| 561 | } |
| 562 | } |
| 563 | |
| 564 | // Dequeues and returns a DecodedTextureBuffer if available, or null otherwise. |
| 565 | public DecodedTextureBuffer dequeueTextureBuffer(int timeoutMs) { |
| 566 | synchronized (newFrameLock) { |
| 567 | if (renderedBuffer == null && timeoutMs > 0 && isWaitingForTexture()) { |
| 568 | try { |
| 569 | newFrameLock.wait(timeoutMs); |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 570 | } catch (InterruptedException e) { |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 571 | // Restore the interrupted status by reinterrupting the thread. |
| 572 | Thread.currentThread().interrupt(); |
| 573 | } |
| 574 | } |
| 575 | DecodedTextureBuffer returnedBuffer = renderedBuffer; |
| 576 | renderedBuffer = null; |
| 577 | return returnedBuffer; |
| 578 | } |
| 579 | } |
| 580 | |
| 581 | public void release() { |
nisse | a44e72c | 2016-05-27 00:27:59 -0700 | [diff] [blame] | 582 | // SurfaceTextureHelper.stopListening() will block until any onTextureFrameAvailable() in |
| 583 | // progress is done. Therefore, the call must be outside any synchronized |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 584 | // statement that is also used in the onTextureFrameAvailable() above to avoid deadlocks. |
nisse | a44e72c | 2016-05-27 00:27:59 -0700 | [diff] [blame] | 585 | surfaceTextureHelper.stopListening(); |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 586 | synchronized (newFrameLock) { |
| 587 | if (renderedBuffer != null) { |
| 588 | surfaceTextureHelper.returnTextureFrame(); |
| 589 | renderedBuffer = null; |
| 590 | } |
| 591 | } |
| 592 | } |
| 593 | } |
| 594 | |
| 595 | // Returns null if no decoded buffer is available, and otherwise a DecodedByteBuffer. |
Magnus Jedvert | 7e31937 | 2015-10-02 15:49:38 +0200 | [diff] [blame] | 596 | // Throws IllegalStateException if call is made on the wrong thread, if color format changes to an |
| 597 | // unsupported format, or if |mediaCodec| is not in the Executing state. Throws CodecException |
| 598 | // upon codec error. |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 599 | private DecodedOutputBuffer dequeueOutputBuffer(int dequeueTimeoutMs) { |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 600 | checkOnMediaCodecThread(); |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 601 | if (decodeStartTimeMs.isEmpty()) { |
| 602 | return null; |
| 603 | } |
Magnus Jedvert | 7e31937 | 2015-10-02 15:49:38 +0200 | [diff] [blame] | 604 | // Drain the decoder until receiving a decoded buffer or hitting |
| 605 | // MediaCodec.INFO_TRY_AGAIN_LATER. |
| 606 | final MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); |
| 607 | while (true) { |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 608 | final int result = |
| 609 | mediaCodec.dequeueOutputBuffer(info, TimeUnit.MILLISECONDS.toMicros(dequeueTimeoutMs)); |
Magnus Jedvert | 7e31937 | 2015-10-02 15:49:38 +0200 | [diff] [blame] | 610 | switch (result) { |
Magnus Jedvert | 7e31937 | 2015-10-02 15:49:38 +0200 | [diff] [blame] | 611 | case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 612 | outputBuffers = mediaCodec.getOutputBuffers(); |
Jiayang Liu | 5975b3c | 2015-09-16 13:40:53 -0700 | [diff] [blame] | 613 | Logging.d(TAG, "Decoder output buffers changed: " + outputBuffers.length); |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 614 | if (hasDecodedFirstFrame) { |
| 615 | throw new RuntimeException("Unexpected output buffer change event."); |
| 616 | } |
Magnus Jedvert | 7e31937 | 2015-10-02 15:49:38 +0200 | [diff] [blame] | 617 | break; |
| 618 | case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 619 | MediaFormat format = mediaCodec.getOutputFormat(); |
Jiayang Liu | 5975b3c | 2015-09-16 13:40:53 -0700 | [diff] [blame] | 620 | Logging.d(TAG, "Decoder format changed: " + format.toString()); |
magjed | 0c29de5 | 2017-03-02 00:55:32 -0800 | [diff] [blame] | 621 | final int newWidth; |
| 622 | final int newHeight; |
| 623 | if (format.containsKey(FORMAT_KEY_CROP_LEFT) && format.containsKey(FORMAT_KEY_CROP_RIGHT) |
| 624 | && format.containsKey(FORMAT_KEY_CROP_BOTTOM) |
| 625 | && format.containsKey(FORMAT_KEY_CROP_TOP)) { |
| 626 | newWidth = 1 + format.getInteger(FORMAT_KEY_CROP_RIGHT) |
| 627 | - format.getInteger(FORMAT_KEY_CROP_LEFT); |
| 628 | newHeight = 1 + format.getInteger(FORMAT_KEY_CROP_BOTTOM) |
| 629 | - format.getInteger(FORMAT_KEY_CROP_TOP); |
| 630 | } else { |
| 631 | newWidth = format.getInteger(MediaFormat.KEY_WIDTH); |
| 632 | newHeight = format.getInteger(MediaFormat.KEY_HEIGHT); |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 633 | } |
magjed | 0c29de5 | 2017-03-02 00:55:32 -0800 | [diff] [blame] | 634 | if (hasDecodedFirstFrame && (newWidth != width || newHeight != height)) { |
| 635 | throw new RuntimeException("Unexpected size change. Configured " + width + "*" + height |
| 636 | + ". New " + newWidth + "*" + newHeight); |
| 637 | } |
| 638 | width = newWidth; |
| 639 | height = newHeight; |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 640 | |
glaznev@webrtc.org | 9967845 | 2014-09-15 17:52:42 +0000 | [diff] [blame] | 641 | if (!useSurface && format.containsKey(MediaFormat.KEY_COLOR_FORMAT)) { |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 642 | colorFormat = format.getInteger(MediaFormat.KEY_COLOR_FORMAT); |
Jiayang Liu | 5975b3c | 2015-09-16 13:40:53 -0700 | [diff] [blame] | 643 | Logging.d(TAG, "Color: 0x" + Integer.toHexString(colorFormat)); |
Magnus Jedvert | 7e31937 | 2015-10-02 15:49:38 +0200 | [diff] [blame] | 644 | if (!supportedColorList.contains(colorFormat)) { |
| 645 | throw new IllegalStateException("Non supported color format: " + colorFormat); |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 646 | } |
| 647 | } |
magjed | 0c29de5 | 2017-03-02 00:55:32 -0800 | [diff] [blame] | 648 | if (format.containsKey(FORMAT_KEY_STRIDE)) { |
| 649 | stride = format.getInteger(FORMAT_KEY_STRIDE); |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 650 | } |
magjed | 0c29de5 | 2017-03-02 00:55:32 -0800 | [diff] [blame] | 651 | if (format.containsKey(FORMAT_KEY_SLICE_HEIGHT)) { |
| 652 | sliceHeight = format.getInteger(FORMAT_KEY_SLICE_HEIGHT); |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 653 | } |
Magnus Jedvert | 7e31937 | 2015-10-02 15:49:38 +0200 | [diff] [blame] | 654 | Logging.d(TAG, "Frame stride and slice height: " + stride + " x " + sliceHeight); |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 655 | stride = Math.max(width, stride); |
| 656 | sliceHeight = Math.max(height, sliceHeight); |
Magnus Jedvert | 7e31937 | 2015-10-02 15:49:38 +0200 | [diff] [blame] | 657 | break; |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 658 | case MediaCodec.INFO_TRY_AGAIN_LATER: |
| 659 | return null; |
Magnus Jedvert | 7e31937 | 2015-10-02 15:49:38 +0200 | [diff] [blame] | 660 | default: |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 661 | hasDecodedFirstFrame = true; |
| 662 | TimeStamps timeStamps = decodeStartTimeMs.remove(); |
magjed | cedddbd | 2016-02-26 09:36:04 -0800 | [diff] [blame] | 663 | long decodeTimeMs = SystemClock.elapsedRealtime() - timeStamps.decodeStartTimeMs; |
| 664 | if (decodeTimeMs > MAX_DECODE_TIME_MS) { |
Alex Glaznev | d570484 | 2016-06-27 11:51:10 -0700 | [diff] [blame] | 665 | Logging.e(TAG, "Very high decode time: " + decodeTimeMs + "ms" |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 666 | + ". Q size: " + decodeStartTimeMs.size() |
| 667 | + ". Might be caused by resuming H264 decoding after a pause."); |
magjed | cedddbd | 2016-02-26 09:36:04 -0800 | [diff] [blame] | 668 | decodeTimeMs = MAX_DECODE_TIME_MS; |
| 669 | } |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 670 | return new DecodedOutputBuffer(result, info.offset, info.size, |
| 671 | TimeUnit.MICROSECONDS.toMillis(info.presentationTimeUs), timeStamps.timeStampMs, |
| 672 | timeStamps.ntpTimeStampMs, decodeTimeMs, SystemClock.elapsedRealtime()); |
| 673 | } |
perkj | 9cb8982 | 2015-11-11 03:27:01 -0800 | [diff] [blame] | 674 | } |
perkj | 9cb8982 | 2015-11-11 03:27:01 -0800 | [diff] [blame] | 675 | } |
| 676 | |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 677 | // Returns null if no decoded buffer is available, and otherwise a DecodedTextureBuffer. |
| 678 | // Throws IllegalStateException if call is made on the wrong thread, if color format changes to an |
| 679 | // unsupported format, or if |mediaCodec| is not in the Executing state. Throws CodecException |
| 680 | // upon codec error. If |dequeueTimeoutMs| > 0, the oldest decoded frame will be dropped if |
| 681 | // a frame can't be returned. |
| 682 | private DecodedTextureBuffer dequeueTextureBuffer(int dequeueTimeoutMs) { |
| 683 | checkOnMediaCodecThread(); |
| 684 | if (!useSurface) { |
| 685 | throw new IllegalStateException("dequeueTexture() called for byte buffer decoding."); |
| 686 | } |
| 687 | DecodedOutputBuffer outputBuffer = dequeueOutputBuffer(dequeueTimeoutMs); |
| 688 | if (outputBuffer != null) { |
| 689 | dequeuedSurfaceOutputBuffers.add(outputBuffer); |
| 690 | } |
| 691 | |
| 692 | MaybeRenderDecodedTextureBuffer(); |
| 693 | // Check if there is texture ready now by waiting max |dequeueTimeoutMs|. |
| 694 | DecodedTextureBuffer renderedBuffer = textureListener.dequeueTextureBuffer(dequeueTimeoutMs); |
| 695 | if (renderedBuffer != null) { |
| 696 | MaybeRenderDecodedTextureBuffer(); |
| 697 | return renderedBuffer; |
| 698 | } |
| 699 | |
| 700 | if ((dequeuedSurfaceOutputBuffers.size() |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 701 | >= Math.min(MAX_QUEUED_OUTPUTBUFFERS, outputBuffers.length) |
| 702 | || (dequeueTimeoutMs > 0 && !dequeuedSurfaceOutputBuffers.isEmpty()))) { |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 703 | ++droppedFrames; |
| 704 | // Drop the oldest frame still in dequeuedSurfaceOutputBuffers. |
| 705 | // The oldest frame is owned by |textureListener| and can't be dropped since |
| 706 | // mediaCodec.releaseOutputBuffer has already been called. |
| 707 | final DecodedOutputBuffer droppedFrame = dequeuedSurfaceOutputBuffers.remove(); |
| 708 | if (dequeueTimeoutMs > 0) { |
perkj | 7baf79f | 2015-11-24 06:26:38 -0800 | [diff] [blame] | 709 | // TODO(perkj): Re-add the below log when VideoRenderGUI has been removed or fixed to |
| 710 | // return the one and only texture even if it does not render. |
glaznev | ae95ff3 | 2016-02-04 11:47:12 -0800 | [diff] [blame] | 711 | Logging.w(TAG, "Draining decoder. Dropping frame with TS: " |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 712 | + droppedFrame.presentationTimeStampMs + ". Total number of dropped frames: " |
| 713 | + droppedFrames); |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 714 | } else { |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 715 | Logging.w(TAG, "Too many output buffers " + dequeuedSurfaceOutputBuffers.size() |
| 716 | + ". Dropping frame with TS: " + droppedFrame.presentationTimeStampMs |
| 717 | + ". Total number of dropped frames: " + droppedFrames); |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 718 | } |
| 719 | |
| 720 | mediaCodec.releaseOutputBuffer(droppedFrame.index, false /* render */); |
sakal | b6760f9 | 2016-09-29 04:12:44 -0700 | [diff] [blame] | 721 | return new DecodedTextureBuffer(0, null, droppedFrame.presentationTimeStampMs, |
| 722 | droppedFrame.timeStampMs, droppedFrame.ntpTimeStampMs, droppedFrame.decodeTimeMs, |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 723 | SystemClock.elapsedRealtime() - droppedFrame.endDecodeTimeMs); |
| 724 | } |
| 725 | return null; |
| 726 | } |
| 727 | |
| 728 | private void MaybeRenderDecodedTextureBuffer() { |
| 729 | if (dequeuedSurfaceOutputBuffers.isEmpty() || textureListener.isWaitingForTexture()) { |
| 730 | return; |
| 731 | } |
| 732 | // Get the first frame in the queue and render to the decoder output surface. |
| 733 | final DecodedOutputBuffer buffer = dequeuedSurfaceOutputBuffers.remove(); |
| 734 | textureListener.addBufferToRender(buffer); |
| 735 | mediaCodec.releaseOutputBuffer(buffer.index, true /* render */); |
| 736 | } |
| 737 | |
magjed | 44bf6f5 | 2015-10-03 02:08:00 -0700 | [diff] [blame] | 738 | // Release a dequeued output byte buffer back to the codec for re-use. Should only be called for |
| 739 | // non-surface decoding. |
| 740 | // Throws IllegalStateException if the call is made on the wrong thread, if codec is configured |
| 741 | // for surface decoding, or if |mediaCodec| is not in the Executing state. Throws |
| 742 | // MediaCodec.CodecException upon codec error. |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 743 | private void returnDecodedOutputBuffer(int index) |
Magnus Jedvert | 7e31937 | 2015-10-02 15:49:38 +0200 | [diff] [blame] | 744 | throws IllegalStateException, MediaCodec.CodecException { |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 745 | checkOnMediaCodecThread(); |
magjed | 44bf6f5 | 2015-10-03 02:08:00 -0700 | [diff] [blame] | 746 | if (useSurface) { |
Per | 488e75f | 2015-11-19 10:43:36 +0100 | [diff] [blame] | 747 | throw new IllegalStateException("returnDecodedOutputBuffer() called for surface decoding."); |
magjed | 44bf6f5 | 2015-10-03 02:08:00 -0700 | [diff] [blame] | 748 | } |
| 749 | mediaCodec.releaseOutputBuffer(index, false /* render */); |
glaznev@webrtc.org | efe4b9a | 2014-07-22 17:44:53 +0000 | [diff] [blame] | 750 | } |
| 751 | } |