glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1 | /* |
| 2 | * libjingle |
| 3 | * Copyright 2015 Google Inc. |
| 4 | * |
| 5 | * Redistribution and use in source and binary forms, with or without |
| 6 | * modification, are permitted provided that the following conditions are met: |
| 7 | * |
| 8 | * 1. Redistributions of source code must retain the above copyright notice, |
| 9 | * this list of conditions and the following disclaimer. |
| 10 | * 2. Redistributions in binary form must reproduce the above copyright notice, |
| 11 | * this list of conditions and the following disclaimer in the documentation |
| 12 | * and/or other materials provided with the distribution. |
| 13 | * 3. The name of the author may not be used to endorse or promote products |
| 14 | * derived from this software without specific prior written permission. |
| 15 | * |
| 16 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED |
| 17 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF |
| 18 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO |
| 19 | * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
| 21 | * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; |
| 22 | * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, |
| 23 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR |
| 24 | * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
| 25 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 26 | * |
| 27 | */ |
| 28 | |
kjellander | a96e2d7 | 2016-02-04 23:52:28 -0800 | [diff] [blame] | 29 | // NOTICE: androidmediaencoder_jni.h must be included before |
| 30 | // androidmediacodeccommon.h to avoid build errors. |
Henrik Kjellander | 15583c1 | 2016-02-10 10:53:12 +0100 | [diff] [blame] | 31 | #include "webrtc/api/java/jni/androidmediaencoder_jni.h" |
| 32 | |
kjellander | a96e2d7 | 2016-02-04 23:52:28 -0800 | [diff] [blame] | 33 | #include "third_party/libyuv/include/libyuv/convert.h" |
| 34 | #include "third_party/libyuv/include/libyuv/convert_from.h" |
| 35 | #include "third_party/libyuv/include/libyuv/video_common.h" |
Henrik Kjellander | 15583c1 | 2016-02-10 10:53:12 +0100 | [diff] [blame] | 36 | #include "webrtc/api/java/jni/androidmediacodeccommon.h" |
| 37 | #include "webrtc/api/java/jni/classreferenceholder.h" |
| 38 | #include "webrtc/api/java/jni/native_handle_impl.h" |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 39 | #include "webrtc/base/bind.h" |
| 40 | #include "webrtc/base/checks.h" |
| 41 | #include "webrtc/base/logging.h" |
| 42 | #include "webrtc/base/thread.h" |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 43 | #include "webrtc/base/thread_checker.h" |
asapersson | 1d61a51 | 2016-01-20 01:13:46 -0800 | [diff] [blame] | 44 | #include "webrtc/common_types.h" |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 45 | #include "webrtc/modules/rtp_rtcp/source/h264_bitstream_parser.h" |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 46 | #include "webrtc/modules/video_coding/include/video_codec_interface.h" |
kjellander@webrtc.org | b7ce964 | 2015-11-18 23:04:10 +0100 | [diff] [blame] | 47 | #include "webrtc/modules/video_coding/utility/quality_scaler.h" |
| 48 | #include "webrtc/modules/video_coding/utility/vp8_header_parser.h" |
Henrik Kjellander | 98f5351 | 2015-10-28 18:17:40 +0100 | [diff] [blame] | 49 | #include "webrtc/system_wrappers/include/field_trial.h" |
| 50 | #include "webrtc/system_wrappers/include/logcat_trace_context.h" |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 51 | |
| 52 | using rtc::Bind; |
| 53 | using rtc::Thread; |
| 54 | using rtc::ThreadManager; |
| 55 | using rtc::scoped_ptr; |
| 56 | |
| 57 | using webrtc::CodecSpecificInfo; |
| 58 | using webrtc::EncodedImage; |
Miguel Casas-Sanchez | 4765070 | 2015-05-29 17:21:40 -0700 | [diff] [blame] | 59 | using webrtc::VideoFrame; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 60 | using webrtc::RTPFragmentationHeader; |
| 61 | using webrtc::VideoCodec; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 62 | using webrtc::VideoCodecType; |
| 63 | using webrtc::kVideoCodecH264; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 64 | using webrtc::kVideoCodecVP8; |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 65 | using webrtc::kVideoCodecVP9; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 66 | |
| 67 | namespace webrtc_jni { |
| 68 | |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 69 | // H.264 start code length. |
| 70 | #define H264_SC_LENGTH 4 |
| 71 | // Maximum allowed NALUs in one output frame. |
| 72 | #define MAX_NALUS_PERFRAME 32 |
| 73 | // Maximum supported HW video encoder resolution. |
| 74 | #define MAX_VIDEO_WIDTH 1280 |
| 75 | #define MAX_VIDEO_HEIGHT 1280 |
| 76 | // Maximum supported HW video encoder fps. |
| 77 | #define MAX_VIDEO_FPS 30 |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 78 | // Maximum allowed fps value in SetRates() call. |
| 79 | #define MAX_ALLOWED_VIDEO_FPS 60 |
| 80 | // Maximum allowed frames in encoder input queue. |
| 81 | #define MAX_ENCODER_Q_SIZE 2 |
| 82 | // Maximum allowed latency in ms. |
| 83 | #define MAX_ENCODER_LATENCY_MS 70 |
glaznev | 919ff75 | 2016-01-27 15:01:03 -0800 | [diff] [blame] | 84 | // Maximum amount of dropped frames caused by full encoder queue - exceeding |
| 85 | // this threshold means that encoder probably got stuck and need to be reset. |
| 86 | #define ENCODER_STALL_FRAMEDROP_THRESHOLD 60 |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 87 | |
| 88 | // Logging macros. |
| 89 | #define TAG_ENCODER "MediaCodecVideoEncoder" |
| 90 | #ifdef TRACK_BUFFER_TIMING |
| 91 | #define ALOGV(...) |
| 92 | __android_log_print(ANDROID_LOG_VERBOSE, TAG_ENCODER, __VA_ARGS__) |
| 93 | #else |
| 94 | #define ALOGV(...) |
| 95 | #endif |
| 96 | #define ALOGD LOG_TAG(rtc::LS_INFO, TAG_ENCODER) |
| 97 | #define ALOGW LOG_TAG(rtc::LS_WARNING, TAG_ENCODER) |
| 98 | #define ALOGE LOG_TAG(rtc::LS_ERROR, TAG_ENCODER) |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 99 | |
asapersson | 1d61a51 | 2016-01-20 01:13:46 -0800 | [diff] [blame] | 100 | namespace { |
| 101 | // Maximum time limit between incoming frames before requesting a key frame. |
| 102 | const size_t kFrameDiffThresholdMs = 1100; |
| 103 | const int kMinKeyFrameInterval = 2; |
| 104 | } // namespace |
| 105 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 106 | // MediaCodecVideoEncoder is a webrtc::VideoEncoder implementation that uses |
| 107 | // Android's MediaCodec SDK API behind the scenes to implement (hopefully) |
| 108 | // HW-backed video encode. This C++ class is implemented as a very thin shim, |
| 109 | // delegating all of the interesting work to org.webrtc.MediaCodecVideoEncoder. |
| 110 | // MediaCodecVideoEncoder is created, operated, and destroyed on a single |
| 111 | // thread, currently the libjingle Worker thread. |
| 112 | class MediaCodecVideoEncoder : public webrtc::VideoEncoder, |
| 113 | public rtc::MessageHandler { |
| 114 | public: |
| 115 | virtual ~MediaCodecVideoEncoder(); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 116 | MediaCodecVideoEncoder(JNIEnv* jni, |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 117 | VideoCodecType codecType, |
| 118 | jobject egl_context); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 119 | |
| 120 | // webrtc::VideoEncoder implementation. Everything trampolines to |
| 121 | // |codec_thread_| for execution. |
| 122 | int32_t InitEncode(const webrtc::VideoCodec* codec_settings, |
| 123 | int32_t /* number_of_cores */, |
| 124 | size_t /* max_payload_size */) override; |
pbos | 22993e1 | 2015-10-19 02:39:06 -0700 | [diff] [blame] | 125 | int32_t Encode(const webrtc::VideoFrame& input_image, |
| 126 | const webrtc::CodecSpecificInfo* /* codec_specific_info */, |
| 127 | const std::vector<webrtc::FrameType>* frame_types) override; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 128 | int32_t RegisterEncodeCompleteCallback( |
| 129 | webrtc::EncodedImageCallback* callback) override; |
| 130 | int32_t Release() override; |
| 131 | int32_t SetChannelParameters(uint32_t /* packet_loss */, |
| 132 | int64_t /* rtt */) override; |
| 133 | int32_t SetRates(uint32_t new_bit_rate, uint32_t frame_rate) override; |
| 134 | |
| 135 | // rtc::MessageHandler implementation. |
| 136 | void OnMessage(rtc::Message* msg) override; |
| 137 | |
jackychen | 61b4d51 | 2015-04-21 15:30:11 -0700 | [diff] [blame] | 138 | void OnDroppedFrame() override; |
| 139 | |
jackychen | 6e2ce6e | 2015-07-13 16:26:33 -0700 | [diff] [blame] | 140 | int GetTargetFramerate() override; |
| 141 | |
Per | ec2922f | 2016-01-27 15:25:46 +0100 | [diff] [blame] | 142 | bool SupportsNativeHandle() const override { return egl_context_ != nullptr; } |
Peter Boström | b7d9a97 | 2015-12-18 16:01:11 +0100 | [diff] [blame] | 143 | const char* ImplementationName() const override; |
| 144 | |
| 145 | private: |
| 146 | // CHECK-fail if not running on |codec_thread_|. |
| 147 | void CheckOnCodecThread(); |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 148 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 149 | private: |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 150 | // ResetCodecOnCodecThread() calls ReleaseOnCodecThread() and |
| 151 | // InitEncodeOnCodecThread() in an attempt to restore the codec to an |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 152 | // operable state. Necessary after all manner of OMX-layer errors. |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 153 | bool ResetCodecOnCodecThread(); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 154 | |
| 155 | // Implementation of webrtc::VideoEncoder methods above, all running on the |
| 156 | // codec thread exclusively. |
| 157 | // |
| 158 | // If width==0 then this is assumed to be a re-initialization and the |
| 159 | // previously-current values are reused instead of the passed parameters |
| 160 | // (makes it easier to reason about thread-safety). |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 161 | int32_t InitEncodeOnCodecThread(int width, int height, int kbps, int fps, |
| 162 | bool use_surface); |
| 163 | // Reconfigure to match |frame| in width, height. Also reconfigures the |
| 164 | // encoder if |frame| is a texture/byte buffer and the encoder is initialized |
| 165 | // for byte buffer/texture. Returns false if reconfiguring fails. |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 166 | bool MaybeReconfigureEncoderOnCodecThread(const webrtc::VideoFrame& frame); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 167 | int32_t EncodeOnCodecThread( |
Miguel Casas-Sanchez | 4765070 | 2015-05-29 17:21:40 -0700 | [diff] [blame] | 168 | const webrtc::VideoFrame& input_image, |
pbos | 22993e1 | 2015-10-19 02:39:06 -0700 | [diff] [blame] | 169 | const std::vector<webrtc::FrameType>* frame_types); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 170 | bool EncodeByteBufferOnCodecThread(JNIEnv* jni, |
| 171 | bool key_frame, const webrtc::VideoFrame& frame, int input_buffer_index); |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 172 | bool EncodeTextureOnCodecThread(JNIEnv* jni, |
| 173 | bool key_frame, const webrtc::VideoFrame& frame); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 174 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 175 | int32_t RegisterEncodeCompleteCallbackOnCodecThread( |
| 176 | webrtc::EncodedImageCallback* callback); |
| 177 | int32_t ReleaseOnCodecThread(); |
| 178 | int32_t SetRatesOnCodecThread(uint32_t new_bit_rate, uint32_t frame_rate); |
| 179 | |
| 180 | // Helper accessors for MediaCodecVideoEncoder$OutputBufferInfo members. |
| 181 | int GetOutputBufferInfoIndex(JNIEnv* jni, jobject j_output_buffer_info); |
| 182 | jobject GetOutputBufferInfoBuffer(JNIEnv* jni, jobject j_output_buffer_info); |
| 183 | bool GetOutputBufferInfoIsKeyFrame(JNIEnv* jni, jobject j_output_buffer_info); |
| 184 | jlong GetOutputBufferInfoPresentationTimestampUs( |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 185 | JNIEnv* jni, jobject j_output_buffer_info); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 186 | |
| 187 | // Deliver any outputs pending in the MediaCodec to our |callback_| and return |
| 188 | // true on success. |
| 189 | bool DeliverPendingOutputs(JNIEnv* jni); |
| 190 | |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 191 | // Search for H.264 start codes. |
| 192 | int32_t NextNaluPosition(uint8_t *buffer, size_t buffer_size); |
| 193 | |
glaznev | 9429148 | 2016-02-01 13:17:18 -0800 | [diff] [blame] | 194 | // Displays encoder statistics. |
| 195 | void LogStatistics(bool force_log); |
| 196 | |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 197 | // Type of video codec. |
| 198 | VideoCodecType codecType_; |
| 199 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 200 | // Valid all the time since RegisterEncodeCompleteCallback() Invoke()s to |
| 201 | // |codec_thread_| synchronously. |
| 202 | webrtc::EncodedImageCallback* callback_; |
| 203 | |
| 204 | // State that is constant for the lifetime of this object once the ctor |
| 205 | // returns. |
| 206 | scoped_ptr<Thread> codec_thread_; // Thread on which to operate MediaCodec. |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 207 | rtc::ThreadChecker codec_thread_checker_; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 208 | ScopedGlobalRef<jclass> j_media_codec_video_encoder_class_; |
| 209 | ScopedGlobalRef<jobject> j_media_codec_video_encoder_; |
| 210 | jmethodID j_init_encode_method_; |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 211 | jmethodID j_get_input_buffers_method_; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 212 | jmethodID j_dequeue_input_buffer_method_; |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 213 | jmethodID j_encode_buffer_method_; |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 214 | jmethodID j_encode_texture_method_; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 215 | jmethodID j_release_method_; |
| 216 | jmethodID j_set_rates_method_; |
| 217 | jmethodID j_dequeue_output_buffer_method_; |
| 218 | jmethodID j_release_output_buffer_method_; |
| 219 | jfieldID j_color_format_field_; |
| 220 | jfieldID j_info_index_field_; |
| 221 | jfieldID j_info_buffer_field_; |
| 222 | jfieldID j_info_is_key_frame_field_; |
| 223 | jfieldID j_info_presentation_timestamp_us_field_; |
| 224 | |
| 225 | // State that is valid only between InitEncode() and the next Release(). |
| 226 | // Touched only on codec_thread_ so no explicit synchronization necessary. |
| 227 | int width_; // Frame width in pixels. |
| 228 | int height_; // Frame height in pixels. |
| 229 | bool inited_; |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 230 | bool use_surface_; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 231 | uint16_t picture_id_; |
| 232 | enum libyuv::FourCC encoder_fourcc_; // Encoder color space format. |
| 233 | int last_set_bitrate_kbps_; // Last-requested bitrate in kbps. |
| 234 | int last_set_fps_; // Last-requested frame rate. |
| 235 | int64_t current_timestamp_us_; // Current frame timestamps in us. |
| 236 | int frames_received_; // Number of frames received by encoder. |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 237 | int frames_encoded_; // Number of frames encoded by encoder. |
glaznev | 919ff75 | 2016-01-27 15:01:03 -0800 | [diff] [blame] | 238 | int frames_dropped_media_encoder_; // Number of frames dropped by encoder. |
| 239 | // Number of dropped frames caused by full queue. |
| 240 | int consecutive_full_queue_frame_drops_; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 241 | int frames_in_queue_; // Number of frames in encoder queue. |
glaznev | 9429148 | 2016-02-01 13:17:18 -0800 | [diff] [blame] | 242 | int64_t stat_start_time_ms_; // Start time for statistics. |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 243 | int current_frames_; // Number of frames in the current statistics interval. |
| 244 | int current_bytes_; // Encoded bytes in the current statistics interval. |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 245 | int current_acc_qp_; // Accumulated QP in the current statistics interval. |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 246 | int current_encoding_time_ms_; // Overall encoding time in the current second |
| 247 | int64_t last_input_timestamp_ms_; // Timestamp of last received yuv frame. |
| 248 | int64_t last_output_timestamp_ms_; // Timestamp of last encoded frame. |
| 249 | std::vector<int32_t> timestamps_; // Video frames timestamp queue. |
| 250 | std::vector<int64_t> render_times_ms_; // Video frames render time queue. |
| 251 | std::vector<int64_t> frame_rtc_times_ms_; // Time when video frame is sent to |
| 252 | // encoder input. |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 253 | int32_t output_timestamp_; // Last output frame timestamp from timestamps_ Q. |
| 254 | int64_t output_render_time_ms_; // Last output frame render time from |
| 255 | // render_times_ms_ queue. |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 256 | // Frame size in bytes fed to MediaCodec. |
| 257 | int yuv_size_; |
| 258 | // True only when between a callback_->Encoded() call return a positive value |
| 259 | // and the next Encode() call being ignored. |
| 260 | bool drop_next_input_frame_; |
| 261 | // Global references; must be deleted in Release(). |
| 262 | std::vector<jobject> input_buffers_; |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 263 | webrtc::QualityScaler quality_scaler_; |
jackychen | 61b4d51 | 2015-04-21 15:30:11 -0700 | [diff] [blame] | 264 | // Dynamic resolution change, off by default. |
| 265 | bool scale_; |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 266 | |
| 267 | // H264 bitstream parser, used to extract QP from encoded bitstreams. |
| 268 | webrtc::H264BitstreamParser h264_bitstream_parser_; |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 269 | |
| 270 | // VP9 variables to populate codec specific structure. |
| 271 | webrtc::GofInfoVP9 gof_; // Contains each frame's temporal information for |
| 272 | // non-flexible VP9 mode. |
| 273 | uint8_t tl0_pic_idx_; |
| 274 | size_t gof_idx_; |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 275 | |
| 276 | // EGL context - owned by factory, should not be allocated/destroyed |
| 277 | // by MediaCodecVideoEncoder. |
| 278 | jobject egl_context_; |
asapersson | 1d61a51 | 2016-01-20 01:13:46 -0800 | [diff] [blame] | 279 | |
| 280 | // Temporary fix for VP8. |
| 281 | // Sends a key frame if frames are largely spaced apart (possibly |
| 282 | // corresponding to a large image change). |
| 283 | int64_t last_frame_received_ms_; |
| 284 | int frames_received_since_last_key_; |
| 285 | webrtc::VideoCodecMode codec_mode_; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 286 | }; |
| 287 | |
| 288 | MediaCodecVideoEncoder::~MediaCodecVideoEncoder() { |
| 289 | // Call Release() to ensure no more callbacks to us after we are deleted. |
| 290 | Release(); |
| 291 | } |
| 292 | |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 293 | MediaCodecVideoEncoder::MediaCodecVideoEncoder( |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 294 | JNIEnv* jni, VideoCodecType codecType, jobject egl_context) : |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 295 | codecType_(codecType), |
| 296 | callback_(NULL), |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 297 | codec_thread_(new Thread()), |
| 298 | j_media_codec_video_encoder_class_( |
| 299 | jni, |
| 300 | FindClass(jni, "org/webrtc/MediaCodecVideoEncoder")), |
| 301 | j_media_codec_video_encoder_( |
| 302 | jni, |
| 303 | jni->NewObject(*j_media_codec_video_encoder_class_, |
| 304 | GetMethodID(jni, |
| 305 | *j_media_codec_video_encoder_class_, |
| 306 | "<init>", |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 307 | "()V"))), |
kjellander | 60ca31b | 2016-01-04 10:15:53 -0800 | [diff] [blame] | 308 | inited_(false), |
| 309 | use_surface_(false), |
| 310 | picture_id_(0), |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 311 | egl_context_(egl_context) { |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 312 | ScopedLocalRefFrame local_ref_frame(jni); |
| 313 | // It would be nice to avoid spinning up a new thread per MediaCodec, and |
| 314 | // instead re-use e.g. the PeerConnectionFactory's |worker_thread_|, but bug |
| 315 | // 2732 means that deadlocks abound. This class synchronously trampolines |
| 316 | // to |codec_thread_|, so if anything else can be coming to _us_ from |
| 317 | // |codec_thread_|, or from any thread holding the |_sendCritSect| described |
| 318 | // in the bug, we have a problem. For now work around that with a dedicated |
| 319 | // thread. |
| 320 | codec_thread_->SetName("MediaCodecVideoEncoder", NULL); |
henrikg | 91d6ede | 2015-09-17 00:24:34 -0700 | [diff] [blame] | 321 | RTC_CHECK(codec_thread_->Start()) << "Failed to start MediaCodecVideoEncoder"; |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 322 | codec_thread_checker_.DetachFromThread(); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 323 | jclass j_output_buffer_info_class = |
| 324 | FindClass(jni, "org/webrtc/MediaCodecVideoEncoder$OutputBufferInfo"); |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 325 | j_init_encode_method_ = GetMethodID( |
| 326 | jni, |
| 327 | *j_media_codec_video_encoder_class_, |
| 328 | "initEncode", |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 329 | "(Lorg/webrtc/MediaCodecVideoEncoder$VideoCodecType;" |
perkj | 48477c1 | 2015-12-18 00:34:37 -0800 | [diff] [blame] | 330 | "IIIILorg/webrtc/EglBase14$Context;)Z"); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 331 | j_get_input_buffers_method_ = GetMethodID( |
| 332 | jni, |
| 333 | *j_media_codec_video_encoder_class_, |
| 334 | "getInputBuffers", |
| 335 | "()[Ljava/nio/ByteBuffer;"); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 336 | j_dequeue_input_buffer_method_ = GetMethodID( |
| 337 | jni, *j_media_codec_video_encoder_class_, "dequeueInputBuffer", "()I"); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 338 | j_encode_buffer_method_ = GetMethodID( |
| 339 | jni, *j_media_codec_video_encoder_class_, "encodeBuffer", "(ZIIJ)Z"); |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 340 | j_encode_texture_method_ = GetMethodID( |
| 341 | jni, *j_media_codec_video_encoder_class_, "encodeTexture", |
| 342 | "(ZI[FJ)Z"); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 343 | j_release_method_ = |
| 344 | GetMethodID(jni, *j_media_codec_video_encoder_class_, "release", "()V"); |
| 345 | j_set_rates_method_ = GetMethodID( |
| 346 | jni, *j_media_codec_video_encoder_class_, "setRates", "(II)Z"); |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 347 | j_dequeue_output_buffer_method_ = GetMethodID( |
| 348 | jni, |
| 349 | *j_media_codec_video_encoder_class_, |
| 350 | "dequeueOutputBuffer", |
| 351 | "()Lorg/webrtc/MediaCodecVideoEncoder$OutputBufferInfo;"); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 352 | j_release_output_buffer_method_ = GetMethodID( |
| 353 | jni, *j_media_codec_video_encoder_class_, "releaseOutputBuffer", "(I)Z"); |
| 354 | |
| 355 | j_color_format_field_ = |
| 356 | GetFieldID(jni, *j_media_codec_video_encoder_class_, "colorFormat", "I"); |
| 357 | j_info_index_field_ = |
| 358 | GetFieldID(jni, j_output_buffer_info_class, "index", "I"); |
| 359 | j_info_buffer_field_ = GetFieldID( |
| 360 | jni, j_output_buffer_info_class, "buffer", "Ljava/nio/ByteBuffer;"); |
| 361 | j_info_is_key_frame_field_ = |
| 362 | GetFieldID(jni, j_output_buffer_info_class, "isKeyFrame", "Z"); |
| 363 | j_info_presentation_timestamp_us_field_ = GetFieldID( |
| 364 | jni, j_output_buffer_info_class, "presentationTimestampUs", "J"); |
| 365 | CHECK_EXCEPTION(jni) << "MediaCodecVideoEncoder ctor failed"; |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 366 | srand(time(NULL)); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 367 | AllowBlockingCalls(); |
| 368 | } |
| 369 | |
| 370 | int32_t MediaCodecVideoEncoder::InitEncode( |
| 371 | const webrtc::VideoCodec* codec_settings, |
| 372 | int32_t /* number_of_cores */, |
| 373 | size_t /* max_payload_size */) { |
jackychen | 61b4d51 | 2015-04-21 15:30:11 -0700 | [diff] [blame] | 374 | const int kMinWidth = 320; |
| 375 | const int kMinHeight = 180; |
jackychen | 98d8cf5 | 2015-05-21 11:12:02 -0700 | [diff] [blame] | 376 | const int kLowQpThresholdDenominator = 3; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 377 | if (codec_settings == NULL) { |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 378 | ALOGE << "NULL VideoCodec instance"; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 379 | return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 380 | } |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 381 | // Factory should guard against other codecs being used with us. |
henrikg | 91d6ede | 2015-09-17 00:24:34 -0700 | [diff] [blame] | 382 | RTC_CHECK(codec_settings->codecType == codecType_) |
| 383 | << "Unsupported codec " << codec_settings->codecType << " for " |
| 384 | << codecType_; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 385 | |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 386 | ALOGD << "InitEncode request"; |
asapersson | 1d61a51 | 2016-01-20 01:13:46 -0800 | [diff] [blame] | 387 | codec_mode_ = codec_settings->mode; |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 388 | scale_ = (codecType_ != kVideoCodecVP9) && (webrtc::field_trial::FindFullName( |
| 389 | "WebRTC-MediaCodecVideoEncoder-AutomaticResize") == "Enabled"); |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 390 | ALOGD << "Encoder automatic resize " << (scale_ ? "enabled" : "disabled"); |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 391 | if (scale_) { |
| 392 | if (codecType_ == kVideoCodecVP8) { |
| 393 | // QP is obtained from VP8-bitstream for HW, so the QP corresponds to the |
| 394 | // (internal) range: [0, 127]. And we cannot change QP_max in HW, so it is |
| 395 | // always = 127. Note that in SW, QP is that of the user-level range [0, |
| 396 | // 63]. |
| 397 | const int kMaxQp = 127; |
glaznev | 919ff75 | 2016-01-27 15:01:03 -0800 | [diff] [blame] | 398 | const int kBadQpThreshold = 95; |
| 399 | quality_scaler_.Init( |
| 400 | kMaxQp / kLowQpThresholdDenominator, kBadQpThreshold, false); |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 401 | } else if (codecType_ == kVideoCodecH264) { |
| 402 | // H264 QP is in the range [0, 51]. |
| 403 | const int kMaxQp = 51; |
Peter Boström | 1741770 | 2015-09-25 17:03:26 +0200 | [diff] [blame] | 404 | const int kBadQpThreshold = 40; |
glaznev | 919ff75 | 2016-01-27 15:01:03 -0800 | [diff] [blame] | 405 | quality_scaler_.Init( |
| 406 | kMaxQp / kLowQpThresholdDenominator, kBadQpThreshold, false); |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 407 | } else { |
| 408 | // When adding codec support to additional hardware codecs, also configure |
| 409 | // their QP thresholds for scaling. |
| 410 | RTC_NOTREACHED() << "Unsupported codec without configured QP thresholds."; |
| 411 | } |
| 412 | quality_scaler_.SetMinResolution(kMinWidth, kMinHeight); |
| 413 | quality_scaler_.ReportFramerate(codec_settings->maxFramerate); |
jackychen | 61b4d51 | 2015-04-21 15:30:11 -0700 | [diff] [blame] | 414 | } |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 415 | return codec_thread_->Invoke<int32_t>( |
| 416 | Bind(&MediaCodecVideoEncoder::InitEncodeOnCodecThread, |
| 417 | this, |
| 418 | codec_settings->width, |
| 419 | codec_settings->height, |
| 420 | codec_settings->startBitrate, |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 421 | codec_settings->maxFramerate, |
| 422 | false /* use_surface */)); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 423 | } |
| 424 | |
| 425 | int32_t MediaCodecVideoEncoder::Encode( |
Miguel Casas-Sanchez | 4765070 | 2015-05-29 17:21:40 -0700 | [diff] [blame] | 426 | const webrtc::VideoFrame& frame, |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 427 | const webrtc::CodecSpecificInfo* /* codec_specific_info */, |
pbos | 22993e1 | 2015-10-19 02:39:06 -0700 | [diff] [blame] | 428 | const std::vector<webrtc::FrameType>* frame_types) { |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 429 | return codec_thread_->Invoke<int32_t>(Bind( |
| 430 | &MediaCodecVideoEncoder::EncodeOnCodecThread, this, frame, frame_types)); |
| 431 | } |
| 432 | |
| 433 | int32_t MediaCodecVideoEncoder::RegisterEncodeCompleteCallback( |
| 434 | webrtc::EncodedImageCallback* callback) { |
| 435 | return codec_thread_->Invoke<int32_t>( |
| 436 | Bind(&MediaCodecVideoEncoder::RegisterEncodeCompleteCallbackOnCodecThread, |
| 437 | this, |
| 438 | callback)); |
| 439 | } |
| 440 | |
| 441 | int32_t MediaCodecVideoEncoder::Release() { |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 442 | ALOGD << "EncoderRelease request"; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 443 | return codec_thread_->Invoke<int32_t>( |
| 444 | Bind(&MediaCodecVideoEncoder::ReleaseOnCodecThread, this)); |
| 445 | } |
| 446 | |
| 447 | int32_t MediaCodecVideoEncoder::SetChannelParameters(uint32_t /* packet_loss */, |
| 448 | int64_t /* rtt */) { |
| 449 | return WEBRTC_VIDEO_CODEC_OK; |
| 450 | } |
| 451 | |
| 452 | int32_t MediaCodecVideoEncoder::SetRates(uint32_t new_bit_rate, |
| 453 | uint32_t frame_rate) { |
| 454 | return codec_thread_->Invoke<int32_t>( |
| 455 | Bind(&MediaCodecVideoEncoder::SetRatesOnCodecThread, |
| 456 | this, |
| 457 | new_bit_rate, |
| 458 | frame_rate)); |
| 459 | } |
| 460 | |
| 461 | void MediaCodecVideoEncoder::OnMessage(rtc::Message* msg) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 462 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 463 | JNIEnv* jni = AttachCurrentThreadIfNeeded(); |
| 464 | ScopedLocalRefFrame local_ref_frame(jni); |
| 465 | |
| 466 | // We only ever send one message to |this| directly (not through a Bind()'d |
| 467 | // functor), so expect no ID/data. |
henrikg | 91d6ede | 2015-09-17 00:24:34 -0700 | [diff] [blame] | 468 | RTC_CHECK(!msg->message_id) << "Unexpected message!"; |
| 469 | RTC_CHECK(!msg->pdata) << "Unexpected message!"; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 470 | if (!inited_) { |
| 471 | return; |
| 472 | } |
| 473 | |
| 474 | // It would be nice to recover from a failure here if one happened, but it's |
| 475 | // unclear how to signal such a failure to the app, so instead we stay silent |
| 476 | // about it and let the next app-called API method reveal the borkedness. |
| 477 | DeliverPendingOutputs(jni); |
| 478 | codec_thread_->PostDelayed(kMediaCodecPollMs, this); |
| 479 | } |
| 480 | |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 481 | bool MediaCodecVideoEncoder::ResetCodecOnCodecThread() { |
| 482 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
| 483 | ALOGE << "ResetOnCodecThread"; |
| 484 | if (ReleaseOnCodecThread() != WEBRTC_VIDEO_CODEC_OK || |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 485 | InitEncodeOnCodecThread(width_, height_, 0, 0, false) != |
| 486 | WEBRTC_VIDEO_CODEC_OK) { |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 487 | // TODO(fischman): wouldn't it be nice if there was a way to gracefully |
| 488 | // degrade to a SW encoder at this point? There isn't one AFAICT :( |
| 489 | // https://code.google.com/p/webrtc/issues/detail?id=2920 |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 490 | return false; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 491 | } |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 492 | return true; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 493 | } |
| 494 | |
| 495 | int32_t MediaCodecVideoEncoder::InitEncodeOnCodecThread( |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 496 | int width, int height, int kbps, int fps, bool use_surface) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 497 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 498 | RTC_CHECK(!use_surface || egl_context_ != nullptr) << "EGL context not set."; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 499 | JNIEnv* jni = AttachCurrentThreadIfNeeded(); |
| 500 | ScopedLocalRefFrame local_ref_frame(jni); |
| 501 | |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 502 | ALOGD << "InitEncodeOnCodecThread Type: " << (int)codecType_ << ", " << |
| 503 | width << " x " << height << ". Bitrate: " << kbps << |
| 504 | " kbps. Fps: " << fps; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 505 | if (kbps == 0) { |
| 506 | kbps = last_set_bitrate_kbps_; |
| 507 | } |
| 508 | if (fps == 0) { |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 509 | fps = MAX_VIDEO_FPS; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 510 | } |
| 511 | |
| 512 | width_ = width; |
| 513 | height_ = height; |
| 514 | last_set_bitrate_kbps_ = kbps; |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 515 | last_set_fps_ = (fps < MAX_VIDEO_FPS) ? fps : MAX_VIDEO_FPS; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 516 | yuv_size_ = width_ * height_ * 3 / 2; |
| 517 | frames_received_ = 0; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 518 | frames_encoded_ = 0; |
glaznev | 919ff75 | 2016-01-27 15:01:03 -0800 | [diff] [blame] | 519 | frames_dropped_media_encoder_ = 0; |
| 520 | consecutive_full_queue_frame_drops_ = 0; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 521 | frames_in_queue_ = 0; |
| 522 | current_timestamp_us_ = 0; |
glaznev | 9429148 | 2016-02-01 13:17:18 -0800 | [diff] [blame] | 523 | stat_start_time_ms_ = GetCurrentTimeMs(); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 524 | current_frames_ = 0; |
| 525 | current_bytes_ = 0; |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 526 | current_acc_qp_ = 0; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 527 | current_encoding_time_ms_ = 0; |
| 528 | last_input_timestamp_ms_ = -1; |
| 529 | last_output_timestamp_ms_ = -1; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 530 | output_timestamp_ = 0; |
| 531 | output_render_time_ms_ = 0; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 532 | timestamps_.clear(); |
| 533 | render_times_ms_.clear(); |
| 534 | frame_rtc_times_ms_.clear(); |
| 535 | drop_next_input_frame_ = false; |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 536 | use_surface_ = use_surface; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 537 | picture_id_ = static_cast<uint16_t>(rand()) & 0x7FFF; |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 538 | gof_.SetGofInfoVP9(webrtc::TemporalStructureMode::kTemporalStructureMode1); |
| 539 | tl0_pic_idx_ = static_cast<uint8_t>(rand()); |
| 540 | gof_idx_ = 0; |
asapersson | 1d61a51 | 2016-01-20 01:13:46 -0800 | [diff] [blame] | 541 | last_frame_received_ms_ = -1; |
| 542 | frames_received_since_last_key_ = kMinKeyFrameInterval; |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 543 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 544 | // We enforce no extra stride/padding in the format creation step. |
Per | ec2922f | 2016-01-27 15:25:46 +0100 | [diff] [blame] | 545 | jobject j_video_codec_enum = JavaEnumFromIndexAndClassName( |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 546 | jni, "MediaCodecVideoEncoder$VideoCodecType", codecType_); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 547 | const bool encode_status = jni->CallBooleanMethod( |
| 548 | *j_media_codec_video_encoder_, j_init_encode_method_, |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 549 | j_video_codec_enum, width, height, kbps, fps, |
| 550 | (use_surface ? egl_context_ : nullptr)); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 551 | if (!encode_status) { |
| 552 | ALOGE << "Failed to configure encoder."; |
| 553 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 554 | } |
| 555 | CHECK_EXCEPTION(jni); |
| 556 | |
Per | 598242a | 2015-11-26 14:28:55 +0100 | [diff] [blame] | 557 | if (!use_surface) { |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 558 | jobjectArray input_buffers = reinterpret_cast<jobjectArray>( |
| 559 | jni->CallObjectMethod(*j_media_codec_video_encoder_, |
| 560 | j_get_input_buffers_method_)); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 561 | CHECK_EXCEPTION(jni); |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 562 | if (IsNull(jni, input_buffers)) { |
| 563 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 564 | } |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 565 | |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 566 | switch (GetIntField(jni, *j_media_codec_video_encoder_, |
| 567 | j_color_format_field_)) { |
| 568 | case COLOR_FormatYUV420Planar: |
| 569 | encoder_fourcc_ = libyuv::FOURCC_YU12; |
| 570 | break; |
| 571 | case COLOR_FormatYUV420SemiPlanar: |
| 572 | case COLOR_QCOM_FormatYUV420SemiPlanar: |
| 573 | case COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m: |
| 574 | encoder_fourcc_ = libyuv::FOURCC_NV12; |
| 575 | break; |
| 576 | default: |
| 577 | LOG(LS_ERROR) << "Wrong color format."; |
| 578 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 579 | } |
| 580 | size_t num_input_buffers = jni->GetArrayLength(input_buffers); |
| 581 | RTC_CHECK(input_buffers_.empty()) |
| 582 | << "Unexpected double InitEncode without Release"; |
| 583 | input_buffers_.resize(num_input_buffers); |
| 584 | for (size_t i = 0; i < num_input_buffers; ++i) { |
| 585 | input_buffers_[i] = |
| 586 | jni->NewGlobalRef(jni->GetObjectArrayElement(input_buffers, i)); |
| 587 | int64_t yuv_buffer_capacity = |
| 588 | jni->GetDirectBufferCapacity(input_buffers_[i]); |
| 589 | CHECK_EXCEPTION(jni); |
| 590 | RTC_CHECK(yuv_buffer_capacity >= yuv_size_) << "Insufficient capacity"; |
| 591 | } |
| 592 | } |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 593 | |
| 594 | inited_ = true; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 595 | codec_thread_->PostDelayed(kMediaCodecPollMs, this); |
| 596 | return WEBRTC_VIDEO_CODEC_OK; |
| 597 | } |
| 598 | |
| 599 | int32_t MediaCodecVideoEncoder::EncodeOnCodecThread( |
Miguel Casas-Sanchez | 4765070 | 2015-05-29 17:21:40 -0700 | [diff] [blame] | 600 | const webrtc::VideoFrame& frame, |
pbos | 22993e1 | 2015-10-19 02:39:06 -0700 | [diff] [blame] | 601 | const std::vector<webrtc::FrameType>* frame_types) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 602 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 603 | JNIEnv* jni = AttachCurrentThreadIfNeeded(); |
| 604 | ScopedLocalRefFrame local_ref_frame(jni); |
| 605 | |
| 606 | if (!inited_) { |
| 607 | return WEBRTC_VIDEO_CODEC_UNINITIALIZED; |
| 608 | } |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 609 | |
asapersson | 1d61a51 | 2016-01-20 01:13:46 -0800 | [diff] [blame] | 610 | bool send_key_frame = false; |
glaznev | 9429148 | 2016-02-01 13:17:18 -0800 | [diff] [blame] | 611 | if (codec_mode_ == webrtc::kRealtimeVideo) { |
asapersson | 1d61a51 | 2016-01-20 01:13:46 -0800 | [diff] [blame] | 612 | ++frames_received_since_last_key_; |
| 613 | int64_t now_ms = GetCurrentTimeMs(); |
| 614 | if (last_frame_received_ms_ != -1 && |
| 615 | (now_ms - last_frame_received_ms_) > kFrameDiffThresholdMs) { |
| 616 | // Add limit to prevent triggering a key for every frame for very low |
| 617 | // framerates (e.g. if frame diff > kFrameDiffThresholdMs). |
| 618 | if (frames_received_since_last_key_ > kMinKeyFrameInterval) { |
| 619 | ALOGD << "Send key, frame diff: " << (now_ms - last_frame_received_ms_); |
| 620 | send_key_frame = true; |
| 621 | } |
| 622 | frames_received_since_last_key_ = 0; |
| 623 | } |
| 624 | last_frame_received_ms_ = now_ms; |
| 625 | } |
| 626 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 627 | frames_received_++; |
| 628 | if (!DeliverPendingOutputs(jni)) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 629 | if (!ResetCodecOnCodecThread()) |
| 630 | return WEBRTC_VIDEO_CODEC_ERROR; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 631 | } |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 632 | if (frames_encoded_ < kMaxEncodedLogFrames) { |
glaznev | 9429148 | 2016-02-01 13:17:18 -0800 | [diff] [blame] | 633 | ALOGD << "Encoder frame in # " << (frames_received_ - 1) << |
| 634 | ". TS: " << (int)(current_timestamp_us_ / 1000) << |
| 635 | ". Q: " << frames_in_queue_ << |
| 636 | ". Fps: " << last_set_fps_ << |
| 637 | ". Kbps: " << last_set_bitrate_kbps_; |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 638 | } |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 639 | |
| 640 | if (drop_next_input_frame_) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 641 | ALOGW << "Encoder drop frame - failed callback."; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 642 | drop_next_input_frame_ = false; |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 643 | current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_; |
glaznev | 919ff75 | 2016-01-27 15:01:03 -0800 | [diff] [blame] | 644 | frames_dropped_media_encoder_++; |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 645 | OnDroppedFrame(); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 646 | return WEBRTC_VIDEO_CODEC_OK; |
| 647 | } |
| 648 | |
henrikg | 91d6ede | 2015-09-17 00:24:34 -0700 | [diff] [blame] | 649 | RTC_CHECK(frame_types->size() == 1) << "Unexpected stream count"; |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 650 | |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 651 | // Check if we accumulated too many frames in encoder input buffers |
| 652 | // or the encoder latency exceeds 70 ms and drop frame if so. |
| 653 | if (frames_in_queue_ > 0 && last_input_timestamp_ms_ >= 0) { |
| 654 | int encoder_latency_ms = last_input_timestamp_ms_ - |
| 655 | last_output_timestamp_ms_; |
| 656 | if (frames_in_queue_ > MAX_ENCODER_Q_SIZE || |
| 657 | encoder_latency_ms > MAX_ENCODER_LATENCY_MS) { |
| 658 | ALOGD << "Drop frame - encoder is behind by " << encoder_latency_ms << |
glaznev | 9429148 | 2016-02-01 13:17:18 -0800 | [diff] [blame] | 659 | " ms. Q size: " << frames_in_queue_ << ". TS: " << |
| 660 | (int)(current_timestamp_us_ / 1000) << ". Fps: " << last_set_fps_ << |
| 661 | ". Consecutive drops: " << consecutive_full_queue_frame_drops_ ; |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 662 | current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_; |
glaznev | 919ff75 | 2016-01-27 15:01:03 -0800 | [diff] [blame] | 663 | consecutive_full_queue_frame_drops_++; |
| 664 | if (consecutive_full_queue_frame_drops_ >= |
| 665 | ENCODER_STALL_FRAMEDROP_THRESHOLD) { |
| 666 | ALOGE << "Encoder got stuck. Reset."; |
| 667 | ResetCodecOnCodecThread(); |
| 668 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 669 | } |
| 670 | frames_dropped_media_encoder_++; |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 671 | OnDroppedFrame(); |
| 672 | return WEBRTC_VIDEO_CODEC_OK; |
| 673 | } |
| 674 | } |
glaznev | 919ff75 | 2016-01-27 15:01:03 -0800 | [diff] [blame] | 675 | consecutive_full_queue_frame_drops_ = 0; |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 676 | |
Per | 598242a | 2015-11-26 14:28:55 +0100 | [diff] [blame] | 677 | VideoFrame input_frame = frame; |
| 678 | if (scale_) { |
| 679 | // Check framerate before spatial resolution change. |
| 680 | quality_scaler_.OnEncodeFrame(frame); |
| 681 | const webrtc::QualityScaler::Resolution scaled_resolution = |
| 682 | quality_scaler_.GetScaledResolution(); |
| 683 | if (scaled_resolution.width != frame.width() || |
| 684 | scaled_resolution.height != frame.height()) { |
| 685 | if (frame.native_handle() != nullptr) { |
| 686 | rtc::scoped_refptr<webrtc::VideoFrameBuffer> scaled_buffer( |
| 687 | static_cast<AndroidTextureBuffer*>( |
Per | 71f5a9a | 2015-12-11 09:32:37 +0100 | [diff] [blame] | 688 | frame.video_frame_buffer().get())->ScaleAndRotate( |
Per | 598242a | 2015-11-26 14:28:55 +0100 | [diff] [blame] | 689 | scaled_resolution.width, |
Per | 71f5a9a | 2015-12-11 09:32:37 +0100 | [diff] [blame] | 690 | scaled_resolution.height, |
| 691 | webrtc::kVideoRotation_0)); |
Per | 598242a | 2015-11-26 14:28:55 +0100 | [diff] [blame] | 692 | input_frame.set_video_frame_buffer(scaled_buffer); |
| 693 | } else { |
| 694 | input_frame = quality_scaler_.GetScaledFrame(frame); |
| 695 | } |
| 696 | } |
| 697 | } |
jackychen | 61b4d51 | 2015-04-21 15:30:11 -0700 | [diff] [blame] | 698 | |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 699 | if (!MaybeReconfigureEncoderOnCodecThread(input_frame)) { |
| 700 | ALOGE << "Failed to reconfigure encoder."; |
| 701 | return WEBRTC_VIDEO_CODEC_ERROR; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 702 | } |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 703 | |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 704 | // Save time when input frame is sent to the encoder input. |
| 705 | frame_rtc_times_ms_.push_back(GetCurrentTimeMs()); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 706 | |
asapersson | 1d61a51 | 2016-01-20 01:13:46 -0800 | [diff] [blame] | 707 | const bool key_frame = |
| 708 | frame_types->front() != webrtc::kVideoFrameDelta || send_key_frame; |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 709 | bool encode_status = true; |
| 710 | if (!input_frame.native_handle()) { |
| 711 | int j_input_buffer_index = jni->CallIntMethod(*j_media_codec_video_encoder_, |
| 712 | j_dequeue_input_buffer_method_); |
| 713 | CHECK_EXCEPTION(jni); |
| 714 | if (j_input_buffer_index == -1) { |
| 715 | // Video codec falls behind - no input buffer available. |
| 716 | ALOGW << "Encoder drop frame - no input buffers available"; |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 717 | frame_rtc_times_ms_.erase(frame_rtc_times_ms_.begin()); |
glaznev | 919ff75 | 2016-01-27 15:01:03 -0800 | [diff] [blame] | 718 | current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_; |
| 719 | frames_dropped_media_encoder_++; |
| 720 | OnDroppedFrame(); |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 721 | return WEBRTC_VIDEO_CODEC_OK; // TODO(fischman): see webrtc bug 2887. |
| 722 | } |
| 723 | if (j_input_buffer_index == -2) { |
| 724 | ResetCodecOnCodecThread(); |
| 725 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 726 | } |
| 727 | encode_status = EncodeByteBufferOnCodecThread(jni, key_frame, input_frame, |
| 728 | j_input_buffer_index); |
| 729 | } else { |
| 730 | encode_status = EncodeTextureOnCodecThread(jni, key_frame, input_frame); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 731 | } |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 732 | |
| 733 | if (!encode_status) { |
| 734 | ALOGE << "Failed encode frame with timestamp: " << input_frame.timestamp(); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 735 | ResetCodecOnCodecThread(); |
perkj | 12f6802 | 2015-10-16 13:31:45 +0200 | [diff] [blame] | 736 | return WEBRTC_VIDEO_CODEC_ERROR; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 737 | } |
| 738 | |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 739 | last_input_timestamp_ms_ = |
| 740 | current_timestamp_us_ / rtc::kNumMicrosecsPerMillisec; |
perkj | 12f6802 | 2015-10-16 13:31:45 +0200 | [diff] [blame] | 741 | frames_in_queue_++; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 742 | |
perkj | 12f6802 | 2015-10-16 13:31:45 +0200 | [diff] [blame] | 743 | // Save input image timestamps for later output |
| 744 | timestamps_.push_back(input_frame.timestamp()); |
| 745 | render_times_ms_.push_back(input_frame.render_time_ms()); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 746 | current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_; |
| 747 | |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 748 | if (!DeliverPendingOutputs(jni)) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 749 | ALOGE << "Failed deliver pending outputs."; |
| 750 | ResetCodecOnCodecThread(); |
| 751 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 752 | } |
| 753 | return WEBRTC_VIDEO_CODEC_OK; |
| 754 | } |
| 755 | |
| 756 | bool MediaCodecVideoEncoder::MaybeReconfigureEncoderOnCodecThread( |
| 757 | const webrtc::VideoFrame& frame) { |
| 758 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
| 759 | |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 760 | const bool is_texture_frame = frame.native_handle() != nullptr; |
| 761 | const bool reconfigure_due_to_format = is_texture_frame != use_surface_; |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 762 | const bool reconfigure_due_to_size = |
| 763 | frame.width() != width_ || frame.height() != height_; |
| 764 | |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 765 | if (reconfigure_due_to_format) { |
| 766 | ALOGD << "Reconfigure encoder due to format change. " |
| 767 | << (use_surface_ ? |
| 768 | "Reconfiguring to encode from byte buffer." : |
| 769 | "Reconfiguring to encode from texture."); |
glaznev | 9429148 | 2016-02-01 13:17:18 -0800 | [diff] [blame] | 770 | LogStatistics(true); |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 771 | } |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 772 | if (reconfigure_due_to_size) { |
glaznev | 9429148 | 2016-02-01 13:17:18 -0800 | [diff] [blame] | 773 | ALOGW << "Reconfigure encoder due to frame resolution change from " |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 774 | << width_ << " x " << height_ << " to " << frame.width() << " x " |
| 775 | << frame.height(); |
glaznev | 9429148 | 2016-02-01 13:17:18 -0800 | [diff] [blame] | 776 | LogStatistics(true); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 777 | width_ = frame.width(); |
| 778 | height_ = frame.height(); |
| 779 | } |
| 780 | |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 781 | if (!reconfigure_due_to_format && !reconfigure_due_to_size) |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 782 | return true; |
| 783 | |
| 784 | ReleaseOnCodecThread(); |
| 785 | |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 786 | return InitEncodeOnCodecThread(width_, height_, 0, 0 , is_texture_frame) == |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 787 | WEBRTC_VIDEO_CODEC_OK; |
| 788 | } |
| 789 | |
| 790 | bool MediaCodecVideoEncoder::EncodeByteBufferOnCodecThread(JNIEnv* jni, |
| 791 | bool key_frame, const webrtc::VideoFrame& frame, int input_buffer_index) { |
| 792 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 793 | RTC_CHECK(!use_surface_); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 794 | |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 795 | jobject j_input_buffer = input_buffers_[input_buffer_index]; |
| 796 | uint8_t* yuv_buffer = |
| 797 | reinterpret_cast<uint8_t*>(jni->GetDirectBufferAddress(j_input_buffer)); |
| 798 | CHECK_EXCEPTION(jni); |
| 799 | RTC_CHECK(yuv_buffer) << "Indirect buffer??"; |
| 800 | RTC_CHECK(!libyuv::ConvertFromI420( |
| 801 | frame.buffer(webrtc::kYPlane), frame.stride(webrtc::kYPlane), |
| 802 | frame.buffer(webrtc::kUPlane), frame.stride(webrtc::kUPlane), |
| 803 | frame.buffer(webrtc::kVPlane), frame.stride(webrtc::kVPlane), |
| 804 | yuv_buffer, width_, width_, height_, encoder_fourcc_)) |
| 805 | << "ConvertFromI420 failed"; |
| 806 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 807 | bool encode_status = jni->CallBooleanMethod(*j_media_codec_video_encoder_, |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 808 | j_encode_buffer_method_, |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 809 | key_frame, |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 810 | input_buffer_index, |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 811 | yuv_size_, |
| 812 | current_timestamp_us_); |
| 813 | CHECK_EXCEPTION(jni); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 814 | return encode_status; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 815 | } |
| 816 | |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 817 | bool MediaCodecVideoEncoder::EncodeTextureOnCodecThread(JNIEnv* jni, |
| 818 | bool key_frame, const webrtc::VideoFrame& frame) { |
| 819 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
| 820 | RTC_CHECK(use_surface_); |
| 821 | NativeHandleImpl* handle = |
| 822 | static_cast<NativeHandleImpl*>(frame.native_handle()); |
| 823 | jfloatArray sampling_matrix = jni->NewFloatArray(16); |
| 824 | jni->SetFloatArrayRegion(sampling_matrix, 0, 16, handle->sampling_matrix); |
| 825 | |
| 826 | bool encode_status = jni->CallBooleanMethod(*j_media_codec_video_encoder_, |
| 827 | j_encode_texture_method_, |
| 828 | key_frame, |
| 829 | handle->oes_texture_id, |
| 830 | sampling_matrix, |
| 831 | current_timestamp_us_); |
| 832 | CHECK_EXCEPTION(jni); |
| 833 | return encode_status; |
| 834 | } |
| 835 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 836 | int32_t MediaCodecVideoEncoder::RegisterEncodeCompleteCallbackOnCodecThread( |
| 837 | webrtc::EncodedImageCallback* callback) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 838 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 839 | JNIEnv* jni = AttachCurrentThreadIfNeeded(); |
| 840 | ScopedLocalRefFrame local_ref_frame(jni); |
| 841 | callback_ = callback; |
| 842 | return WEBRTC_VIDEO_CODEC_OK; |
| 843 | } |
| 844 | |
| 845 | int32_t MediaCodecVideoEncoder::ReleaseOnCodecThread() { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 846 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 847 | if (!inited_) { |
| 848 | return WEBRTC_VIDEO_CODEC_OK; |
| 849 | } |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 850 | JNIEnv* jni = AttachCurrentThreadIfNeeded(); |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 851 | ALOGD << "EncoderReleaseOnCodecThread: Frames received: " << |
| 852 | frames_received_ << ". Encoded: " << frames_encoded_ << |
glaznev | 919ff75 | 2016-01-27 15:01:03 -0800 | [diff] [blame] | 853 | ". Dropped: " << frames_dropped_media_encoder_; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 854 | ScopedLocalRefFrame local_ref_frame(jni); |
| 855 | for (size_t i = 0; i < input_buffers_.size(); ++i) |
| 856 | jni->DeleteGlobalRef(input_buffers_[i]); |
| 857 | input_buffers_.clear(); |
| 858 | jni->CallVoidMethod(*j_media_codec_video_encoder_, j_release_method_); |
| 859 | CHECK_EXCEPTION(jni); |
| 860 | rtc::MessageQueueManager::Clear(this); |
| 861 | inited_ = false; |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 862 | use_surface_ = false; |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 863 | ALOGD << "EncoderReleaseOnCodecThread done."; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 864 | return WEBRTC_VIDEO_CODEC_OK; |
| 865 | } |
| 866 | |
| 867 | int32_t MediaCodecVideoEncoder::SetRatesOnCodecThread(uint32_t new_bit_rate, |
| 868 | uint32_t frame_rate) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 869 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 870 | frame_rate = (frame_rate < MAX_ALLOWED_VIDEO_FPS) ? |
| 871 | frame_rate : MAX_ALLOWED_VIDEO_FPS; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 872 | if (last_set_bitrate_kbps_ == new_bit_rate && |
| 873 | last_set_fps_ == frame_rate) { |
| 874 | return WEBRTC_VIDEO_CODEC_OK; |
| 875 | } |
glaznev | 919ff75 | 2016-01-27 15:01:03 -0800 | [diff] [blame] | 876 | if (scale_) { |
| 877 | quality_scaler_.ReportFramerate(frame_rate); |
| 878 | } |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 879 | JNIEnv* jni = AttachCurrentThreadIfNeeded(); |
| 880 | ScopedLocalRefFrame local_ref_frame(jni); |
| 881 | if (new_bit_rate > 0) { |
| 882 | last_set_bitrate_kbps_ = new_bit_rate; |
| 883 | } |
| 884 | if (frame_rate > 0) { |
| 885 | last_set_fps_ = frame_rate; |
| 886 | } |
| 887 | bool ret = jni->CallBooleanMethod(*j_media_codec_video_encoder_, |
| 888 | j_set_rates_method_, |
| 889 | last_set_bitrate_kbps_, |
| 890 | last_set_fps_); |
| 891 | CHECK_EXCEPTION(jni); |
| 892 | if (!ret) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 893 | ResetCodecOnCodecThread(); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 894 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 895 | } |
| 896 | return WEBRTC_VIDEO_CODEC_OK; |
| 897 | } |
| 898 | |
| 899 | int MediaCodecVideoEncoder::GetOutputBufferInfoIndex( |
| 900 | JNIEnv* jni, |
| 901 | jobject j_output_buffer_info) { |
| 902 | return GetIntField(jni, j_output_buffer_info, j_info_index_field_); |
| 903 | } |
| 904 | |
| 905 | jobject MediaCodecVideoEncoder::GetOutputBufferInfoBuffer( |
| 906 | JNIEnv* jni, |
| 907 | jobject j_output_buffer_info) { |
| 908 | return GetObjectField(jni, j_output_buffer_info, j_info_buffer_field_); |
| 909 | } |
| 910 | |
| 911 | bool MediaCodecVideoEncoder::GetOutputBufferInfoIsKeyFrame( |
| 912 | JNIEnv* jni, |
| 913 | jobject j_output_buffer_info) { |
| 914 | return GetBooleanField(jni, j_output_buffer_info, j_info_is_key_frame_field_); |
| 915 | } |
| 916 | |
| 917 | jlong MediaCodecVideoEncoder::GetOutputBufferInfoPresentationTimestampUs( |
| 918 | JNIEnv* jni, |
| 919 | jobject j_output_buffer_info) { |
| 920 | return GetLongField( |
| 921 | jni, j_output_buffer_info, j_info_presentation_timestamp_us_field_); |
| 922 | } |
| 923 | |
| 924 | bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 925 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 926 | while (true) { |
| 927 | jobject j_output_buffer_info = jni->CallObjectMethod( |
| 928 | *j_media_codec_video_encoder_, j_dequeue_output_buffer_method_); |
| 929 | CHECK_EXCEPTION(jni); |
| 930 | if (IsNull(jni, j_output_buffer_info)) { |
| 931 | break; |
| 932 | } |
| 933 | |
| 934 | int output_buffer_index = |
| 935 | GetOutputBufferInfoIndex(jni, j_output_buffer_info); |
| 936 | if (output_buffer_index == -1) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 937 | ResetCodecOnCodecThread(); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 938 | return false; |
| 939 | } |
| 940 | |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 941 | // Get key and config frame flags. |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 942 | jobject j_output_buffer = |
| 943 | GetOutputBufferInfoBuffer(jni, j_output_buffer_info); |
| 944 | bool key_frame = GetOutputBufferInfoIsKeyFrame(jni, j_output_buffer_info); |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 945 | |
| 946 | // Get frame timestamps from a queue - for non config frames only. |
| 947 | int64_t frame_encoding_time_ms = 0; |
| 948 | last_output_timestamp_ms_ = |
| 949 | GetOutputBufferInfoPresentationTimestampUs(jni, j_output_buffer_info) / |
| 950 | 1000; |
| 951 | if (frames_in_queue_ > 0) { |
| 952 | output_timestamp_ = timestamps_.front(); |
| 953 | timestamps_.erase(timestamps_.begin()); |
| 954 | output_render_time_ms_ = render_times_ms_.front(); |
| 955 | render_times_ms_.erase(render_times_ms_.begin()); |
| 956 | frame_encoding_time_ms = GetCurrentTimeMs() - frame_rtc_times_ms_.front(); |
| 957 | frame_rtc_times_ms_.erase(frame_rtc_times_ms_.begin()); |
| 958 | frames_in_queue_--; |
| 959 | } |
| 960 | |
| 961 | // Extract payload. |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 962 | size_t payload_size = jni->GetDirectBufferCapacity(j_output_buffer); |
Peter Boström | 0c4e06b | 2015-10-07 12:23:21 +0200 | [diff] [blame] | 963 | uint8_t* payload = reinterpret_cast<uint8_t*>( |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 964 | jni->GetDirectBufferAddress(j_output_buffer)); |
| 965 | CHECK_EXCEPTION(jni); |
| 966 | |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 967 | if (frames_encoded_ < kMaxEncodedLogFrames) { |
glaznev | 9429148 | 2016-02-01 13:17:18 -0800 | [diff] [blame] | 968 | int current_latency = |
| 969 | (int)(last_input_timestamp_ms_ - last_output_timestamp_ms_); |
| 970 | ALOGD << "Encoder frame out # " << frames_encoded_ << |
| 971 | ". Key: " << key_frame << |
| 972 | ". Size: " << payload_size << |
| 973 | ". TS: " << (int)last_output_timestamp_ms_ << |
| 974 | ". Latency: " << current_latency << |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 975 | ". EncTime: " << frame_encoding_time_ms; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 976 | } |
| 977 | |
| 978 | // Callback - return encoded frame. |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 979 | int32_t callback_status = 0; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 980 | if (callback_) { |
| 981 | scoped_ptr<webrtc::EncodedImage> image( |
| 982 | new webrtc::EncodedImage(payload, payload_size, payload_size)); |
| 983 | image->_encodedWidth = width_; |
| 984 | image->_encodedHeight = height_; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 985 | image->_timeStamp = output_timestamp_; |
| 986 | image->capture_time_ms_ = output_render_time_ms_; |
Peter Boström | 49e196a | 2015-10-23 15:58:18 +0200 | [diff] [blame] | 987 | image->_frameType = |
| 988 | (key_frame ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 989 | image->_completeFrame = true; |
asapersson | 075fb4b | 2015-10-29 08:49:14 -0700 | [diff] [blame] | 990 | image->adapt_reason_.quality_resolution_downscales = |
| 991 | scale_ ? quality_scaler_.downscale_shift() : -1; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 992 | |
| 993 | webrtc::CodecSpecificInfo info; |
| 994 | memset(&info, 0, sizeof(info)); |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 995 | info.codecType = codecType_; |
| 996 | if (codecType_ == kVideoCodecVP8) { |
| 997 | info.codecSpecific.VP8.pictureId = picture_id_; |
| 998 | info.codecSpecific.VP8.nonReference = false; |
| 999 | info.codecSpecific.VP8.simulcastIdx = 0; |
| 1000 | info.codecSpecific.VP8.temporalIdx = webrtc::kNoTemporalIdx; |
| 1001 | info.codecSpecific.VP8.layerSync = false; |
| 1002 | info.codecSpecific.VP8.tl0PicIdx = webrtc::kNoTl0PicIdx; |
| 1003 | info.codecSpecific.VP8.keyIdx = webrtc::kNoKeyIdx; |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 1004 | } else if (codecType_ == kVideoCodecVP9) { |
| 1005 | if (key_frame) { |
| 1006 | gof_idx_ = 0; |
| 1007 | } |
| 1008 | info.codecSpecific.VP9.picture_id = picture_id_; |
| 1009 | info.codecSpecific.VP9.inter_pic_predicted = key_frame ? false : true; |
| 1010 | info.codecSpecific.VP9.flexible_mode = false; |
| 1011 | info.codecSpecific.VP9.ss_data_available = key_frame ? true : false; |
| 1012 | info.codecSpecific.VP9.tl0_pic_idx = tl0_pic_idx_++; |
| 1013 | info.codecSpecific.VP9.temporal_idx = webrtc::kNoTemporalIdx; |
| 1014 | info.codecSpecific.VP9.spatial_idx = webrtc::kNoSpatialIdx; |
| 1015 | info.codecSpecific.VP9.temporal_up_switch = true; |
| 1016 | info.codecSpecific.VP9.inter_layer_predicted = false; |
| 1017 | info.codecSpecific.VP9.gof_idx = |
| 1018 | static_cast<uint8_t>(gof_idx_++ % gof_.num_frames_in_gof); |
| 1019 | info.codecSpecific.VP9.num_spatial_layers = 1; |
| 1020 | info.codecSpecific.VP9.spatial_layer_resolution_present = false; |
| 1021 | if (info.codecSpecific.VP9.ss_data_available) { |
| 1022 | info.codecSpecific.VP9.spatial_layer_resolution_present = true; |
| 1023 | info.codecSpecific.VP9.width[0] = width_; |
| 1024 | info.codecSpecific.VP9.height[0] = height_; |
| 1025 | info.codecSpecific.VP9.gof.CopyGofInfoVP9(gof_); |
| 1026 | } |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1027 | } |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 1028 | picture_id_ = (picture_id_ + 1) & 0x7FFF; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1029 | |
| 1030 | // Generate a header describing a single fragment. |
| 1031 | webrtc::RTPFragmentationHeader header; |
| 1032 | memset(&header, 0, sizeof(header)); |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 1033 | if (codecType_ == kVideoCodecVP8 || codecType_ == kVideoCodecVP9) { |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1034 | header.VerifyAndAllocateFragmentationHeader(1); |
| 1035 | header.fragmentationOffset[0] = 0; |
| 1036 | header.fragmentationLength[0] = image->_length; |
| 1037 | header.fragmentationPlType[0] = 0; |
| 1038 | header.fragmentationTimeDiff[0] = 0; |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 1039 | if (codecType_ == kVideoCodecVP8 && scale_) { |
asapersson | 86b0160 | 2015-10-20 23:55:26 -0700 | [diff] [blame] | 1040 | int qp; |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 1041 | if (webrtc::vp8::GetQp(payload, payload_size, &qp)) { |
| 1042 | current_acc_qp_ += qp; |
asapersson | 86b0160 | 2015-10-20 23:55:26 -0700 | [diff] [blame] | 1043 | quality_scaler_.ReportQP(qp); |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 1044 | } |
asapersson | 86b0160 | 2015-10-20 23:55:26 -0700 | [diff] [blame] | 1045 | } |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1046 | } else if (codecType_ == kVideoCodecH264) { |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 1047 | if (scale_) { |
| 1048 | h264_bitstream_parser_.ParseBitstream(payload, payload_size); |
| 1049 | int qp; |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 1050 | if (h264_bitstream_parser_.GetLastSliceQp(&qp)) { |
| 1051 | current_acc_qp_ += qp; |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 1052 | quality_scaler_.ReportQP(qp); |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 1053 | } |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 1054 | } |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1055 | // For H.264 search for start codes. |
| 1056 | int32_t scPositions[MAX_NALUS_PERFRAME + 1] = {}; |
| 1057 | int32_t scPositionsLength = 0; |
| 1058 | int32_t scPosition = 0; |
| 1059 | while (scPositionsLength < MAX_NALUS_PERFRAME) { |
| 1060 | int32_t naluPosition = NextNaluPosition( |
| 1061 | payload + scPosition, payload_size - scPosition); |
| 1062 | if (naluPosition < 0) { |
| 1063 | break; |
| 1064 | } |
| 1065 | scPosition += naluPosition; |
| 1066 | scPositions[scPositionsLength++] = scPosition; |
| 1067 | scPosition += H264_SC_LENGTH; |
| 1068 | } |
| 1069 | if (scPositionsLength == 0) { |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 1070 | ALOGE << "Start code is not found!"; |
| 1071 | ALOGE << "Data:" << image->_buffer[0] << " " << image->_buffer[1] |
| 1072 | << " " << image->_buffer[2] << " " << image->_buffer[3] |
| 1073 | << " " << image->_buffer[4] << " " << image->_buffer[5]; |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 1074 | ResetCodecOnCodecThread(); |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1075 | return false; |
| 1076 | } |
| 1077 | scPositions[scPositionsLength] = payload_size; |
| 1078 | header.VerifyAndAllocateFragmentationHeader(scPositionsLength); |
| 1079 | for (size_t i = 0; i < scPositionsLength; i++) { |
| 1080 | header.fragmentationOffset[i] = scPositions[i] + H264_SC_LENGTH; |
| 1081 | header.fragmentationLength[i] = |
| 1082 | scPositions[i + 1] - header.fragmentationOffset[i]; |
| 1083 | header.fragmentationPlType[i] = 0; |
| 1084 | header.fragmentationTimeDiff[i] = 0; |
| 1085 | } |
| 1086 | } |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1087 | |
| 1088 | callback_status = callback_->Encoded(*image, &info, &header); |
| 1089 | } |
| 1090 | |
| 1091 | // Return output buffer back to the encoder. |
| 1092 | bool success = jni->CallBooleanMethod(*j_media_codec_video_encoder_, |
| 1093 | j_release_output_buffer_method_, |
| 1094 | output_buffer_index); |
| 1095 | CHECK_EXCEPTION(jni); |
| 1096 | if (!success) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 1097 | ResetCodecOnCodecThread(); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1098 | return false; |
| 1099 | } |
| 1100 | |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 1101 | // Calculate and print encoding statistics - every 3 seconds. |
| 1102 | frames_encoded_++; |
| 1103 | current_frames_++; |
| 1104 | current_bytes_ += payload_size; |
| 1105 | current_encoding_time_ms_ += frame_encoding_time_ms; |
glaznev | 9429148 | 2016-02-01 13:17:18 -0800 | [diff] [blame] | 1106 | LogStatistics(false); |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 1107 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1108 | if (callback_status > 0) { |
| 1109 | drop_next_input_frame_ = true; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1110 | // Theoretically could handle callback_status<0 here, but unclear what |
| 1111 | // that would mean for us. |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1112 | } |
| 1113 | } |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1114 | return true; |
| 1115 | } |
| 1116 | |
glaznev | 9429148 | 2016-02-01 13:17:18 -0800 | [diff] [blame] | 1117 | void MediaCodecVideoEncoder::LogStatistics(bool force_log) { |
| 1118 | int statistic_time_ms = GetCurrentTimeMs() - stat_start_time_ms_; |
| 1119 | if ((statistic_time_ms >= kMediaCodecStatisticsIntervalMs || force_log) && |
| 1120 | current_frames_ > 0 && statistic_time_ms > 0) { |
| 1121 | int current_bitrate = current_bytes_ * 8 / statistic_time_ms; |
| 1122 | int current_fps = |
| 1123 | (current_frames_ * 1000 + statistic_time_ms / 2) / statistic_time_ms; |
| 1124 | ALOGD << "Encoded frames: " << frames_encoded_ << |
| 1125 | ". Bitrate: " << current_bitrate << |
| 1126 | ", target: " << last_set_bitrate_kbps_ << " kbps" << |
| 1127 | ", fps: " << current_fps << |
| 1128 | ", encTime: " << (current_encoding_time_ms_ / current_frames_) << |
| 1129 | ". QP: " << (current_acc_qp_ / current_frames_) << |
| 1130 | " for last " << statistic_time_ms << " ms."; |
| 1131 | stat_start_time_ms_ = GetCurrentTimeMs(); |
| 1132 | current_frames_ = 0; |
| 1133 | current_bytes_ = 0; |
| 1134 | current_acc_qp_ = 0; |
| 1135 | current_encoding_time_ms_ = 0; |
| 1136 | } |
| 1137 | } |
| 1138 | |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1139 | int32_t MediaCodecVideoEncoder::NextNaluPosition( |
| 1140 | uint8_t *buffer, size_t buffer_size) { |
| 1141 | if (buffer_size < H264_SC_LENGTH) { |
| 1142 | return -1; |
| 1143 | } |
| 1144 | uint8_t *head = buffer; |
| 1145 | // Set end buffer pointer to 4 bytes before actual buffer end so we can |
| 1146 | // access head[1], head[2] and head[3] in a loop without buffer overrun. |
| 1147 | uint8_t *end = buffer + buffer_size - H264_SC_LENGTH; |
| 1148 | |
| 1149 | while (head < end) { |
| 1150 | if (head[0]) { |
| 1151 | head++; |
| 1152 | continue; |
| 1153 | } |
| 1154 | if (head[1]) { // got 00xx |
| 1155 | head += 2; |
| 1156 | continue; |
| 1157 | } |
| 1158 | if (head[2]) { // got 0000xx |
| 1159 | head += 3; |
| 1160 | continue; |
| 1161 | } |
| 1162 | if (head[3] != 0x01) { // got 000000xx |
glaznev@webrtc.org | dc08a23 | 2015-03-06 23:32:20 +0000 | [diff] [blame] | 1163 | head++; // xx != 1, continue searching. |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1164 | continue; |
| 1165 | } |
| 1166 | return (int32_t)(head - buffer); |
| 1167 | } |
| 1168 | return -1; |
| 1169 | } |
| 1170 | |
jackychen | 61b4d51 | 2015-04-21 15:30:11 -0700 | [diff] [blame] | 1171 | void MediaCodecVideoEncoder::OnDroppedFrame() { |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 1172 | // Report dropped frame to quality_scaler_. |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 1173 | if (scale_) |
| 1174 | quality_scaler_.ReportDroppedFrame(); |
jackychen | 61b4d51 | 2015-04-21 15:30:11 -0700 | [diff] [blame] | 1175 | } |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1176 | |
jackychen | 6e2ce6e | 2015-07-13 16:26:33 -0700 | [diff] [blame] | 1177 | int MediaCodecVideoEncoder::GetTargetFramerate() { |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 1178 | return scale_ ? quality_scaler_.GetTargetFramerate() : -1; |
jackychen | 6e2ce6e | 2015-07-13 16:26:33 -0700 | [diff] [blame] | 1179 | } |
| 1180 | |
Peter Boström | b7d9a97 | 2015-12-18 16:01:11 +0100 | [diff] [blame] | 1181 | const char* MediaCodecVideoEncoder::ImplementationName() const { |
| 1182 | return "MediaCodec"; |
| 1183 | } |
| 1184 | |
Per | ec2922f | 2016-01-27 15:25:46 +0100 | [diff] [blame] | 1185 | MediaCodecVideoEncoderFactory::MediaCodecVideoEncoderFactory() { |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1186 | JNIEnv* jni = AttachCurrentThreadIfNeeded(); |
| 1187 | ScopedLocalRefFrame local_ref_frame(jni); |
| 1188 | jclass j_encoder_class = FindClass(jni, "org/webrtc/MediaCodecVideoEncoder"); |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1189 | supported_codecs_.clear(); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1190 | |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1191 | bool is_vp8_hw_supported = jni->CallStaticBooleanMethod( |
| 1192 | j_encoder_class, |
| 1193 | GetStaticMethodID(jni, j_encoder_class, "isVp8HwSupported", "()Z")); |
| 1194 | CHECK_EXCEPTION(jni); |
| 1195 | if (is_vp8_hw_supported) { |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 1196 | ALOGD << "VP8 HW Encoder supported."; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1197 | supported_codecs_.push_back(VideoCodec(kVideoCodecVP8, "VP8", |
| 1198 | MAX_VIDEO_WIDTH, MAX_VIDEO_HEIGHT, MAX_VIDEO_FPS)); |
| 1199 | } |
| 1200 | |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 1201 | bool is_vp9_hw_supported = jni->CallStaticBooleanMethod( |
| 1202 | j_encoder_class, |
| 1203 | GetStaticMethodID(jni, j_encoder_class, "isVp9HwSupported", "()Z")); |
| 1204 | CHECK_EXCEPTION(jni); |
| 1205 | if (is_vp9_hw_supported) { |
| 1206 | ALOGD << "VP9 HW Encoder supported."; |
| 1207 | supported_codecs_.push_back(VideoCodec(kVideoCodecVP9, "VP9", |
| 1208 | MAX_VIDEO_WIDTH, MAX_VIDEO_HEIGHT, MAX_VIDEO_FPS)); |
| 1209 | } |
| 1210 | |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1211 | bool is_h264_hw_supported = jni->CallStaticBooleanMethod( |
| 1212 | j_encoder_class, |
| 1213 | GetStaticMethodID(jni, j_encoder_class, "isH264HwSupported", "()Z")); |
| 1214 | CHECK_EXCEPTION(jni); |
| 1215 | if (is_h264_hw_supported) { |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 1216 | ALOGD << "H.264 HW Encoder supported."; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1217 | supported_codecs_.push_back(VideoCodec(kVideoCodecH264, "H264", |
| 1218 | MAX_VIDEO_WIDTH, MAX_VIDEO_HEIGHT, MAX_VIDEO_FPS)); |
| 1219 | } |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1220 | } |
| 1221 | |
Per | ec2922f | 2016-01-27 15:25:46 +0100 | [diff] [blame] | 1222 | MediaCodecVideoEncoderFactory::~MediaCodecVideoEncoderFactory() { |
| 1223 | ALOGD << "MediaCodecVideoEncoderFactory dtor"; |
| 1224 | } |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1225 | |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 1226 | void MediaCodecVideoEncoderFactory::SetEGLContext( |
| 1227 | JNIEnv* jni, jobject render_egl_context) { |
| 1228 | ALOGD << "MediaCodecVideoEncoderFactory::SetEGLContext"; |
Per | ec2922f | 2016-01-27 15:25:46 +0100 | [diff] [blame] | 1229 | if (!egl_base_.CreateEglBase(jni, render_egl_context)) { |
| 1230 | ALOGW << "Invalid EGL context - HW surface encoding is disabled."; |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 1231 | } |
| 1232 | } |
| 1233 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1234 | webrtc::VideoEncoder* MediaCodecVideoEncoderFactory::CreateVideoEncoder( |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1235 | VideoCodecType type) { |
| 1236 | if (supported_codecs_.empty()) { |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 1237 | ALOGW << "No HW video encoder for type " << (int)type; |
Per | ec2922f | 2016-01-27 15:25:46 +0100 | [diff] [blame] | 1238 | return nullptr; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1239 | } |
| 1240 | for (std::vector<VideoCodec>::const_iterator it = supported_codecs_.begin(); |
| 1241 | it != supported_codecs_.end(); ++it) { |
| 1242 | if (it->type == type) { |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 1243 | ALOGD << "Create HW video encoder for type " << (int)type << |
| 1244 | " (" << it->name << ")."; |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 1245 | return new MediaCodecVideoEncoder(AttachCurrentThreadIfNeeded(), type, |
Per | ec2922f | 2016-01-27 15:25:46 +0100 | [diff] [blame] | 1246 | egl_base_.egl_base_context()); |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1247 | } |
| 1248 | } |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 1249 | ALOGW << "Can not find HW video encoder for type " << (int)type; |
Per | ec2922f | 2016-01-27 15:25:46 +0100 | [diff] [blame] | 1250 | return nullptr; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1251 | } |
| 1252 | |
| 1253 | const std::vector<MediaCodecVideoEncoderFactory::VideoCodec>& |
| 1254 | MediaCodecVideoEncoderFactory::codecs() const { |
| 1255 | return supported_codecs_; |
| 1256 | } |
| 1257 | |
| 1258 | void MediaCodecVideoEncoderFactory::DestroyVideoEncoder( |
| 1259 | webrtc::VideoEncoder* encoder) { |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 1260 | ALOGD << "Destroy video encoder."; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1261 | delete encoder; |
| 1262 | } |
| 1263 | |
| 1264 | } // namespace webrtc_jni |
| 1265 | |