glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1 | /* |
kjellander | b24317b | 2016-02-10 07:54:43 -0800 | [diff] [blame] | 2 | * Copyright 2015 The WebRTC project authors. All Rights Reserved. |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 3 | * |
kjellander | b24317b | 2016-02-10 07:54:43 -0800 | [diff] [blame] | 4 | * Use of this source code is governed by a BSD-style license |
| 5 | * that can be found in the LICENSE file in the root of the source |
| 6 | * tree. An additional intellectual property rights grant can be found |
| 7 | * in the file PATENTS. All contributing project authors may |
| 8 | * be found in the AUTHORS file in the root of the source tree. |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 9 | */ |
| 10 | |
kjellander | a96e2d7 | 2016-02-04 23:52:28 -0800 | [diff] [blame] | 11 | // NOTICE: androidmediaencoder_jni.h must be included before |
| 12 | // androidmediacodeccommon.h to avoid build errors. |
Henrik Kjellander | 15583c1 | 2016-02-10 10:53:12 +0100 | [diff] [blame] | 13 | #include "webrtc/api/java/jni/androidmediaencoder_jni.h" |
| 14 | |
Per | ba7dc72 | 2016-04-19 15:01:23 +0200 | [diff] [blame] | 15 | #include <algorithm> |
| 16 | #include <list> |
| 17 | |
kjellander | a96e2d7 | 2016-02-04 23:52:28 -0800 | [diff] [blame] | 18 | #include "third_party/libyuv/include/libyuv/convert.h" |
| 19 | #include "third_party/libyuv/include/libyuv/convert_from.h" |
| 20 | #include "third_party/libyuv/include/libyuv/video_common.h" |
Henrik Kjellander | 15583c1 | 2016-02-10 10:53:12 +0100 | [diff] [blame] | 21 | #include "webrtc/api/java/jni/androidmediacodeccommon.h" |
| 22 | #include "webrtc/api/java/jni/classreferenceholder.h" |
| 23 | #include "webrtc/api/java/jni/native_handle_impl.h" |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 24 | #include "webrtc/base/bind.h" |
| 25 | #include "webrtc/base/checks.h" |
| 26 | #include "webrtc/base/logging.h" |
| 27 | #include "webrtc/base/thread.h" |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 28 | #include "webrtc/base/thread_checker.h" |
asapersson | 1d61a51 | 2016-01-20 01:13:46 -0800 | [diff] [blame] | 29 | #include "webrtc/common_types.h" |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 30 | #include "webrtc/modules/rtp_rtcp/source/h264_bitstream_parser.h" |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 31 | #include "webrtc/modules/video_coding/include/video_codec_interface.h" |
kjellander@webrtc.org | b7ce964 | 2015-11-18 23:04:10 +0100 | [diff] [blame] | 32 | #include "webrtc/modules/video_coding/utility/quality_scaler.h" |
| 33 | #include "webrtc/modules/video_coding/utility/vp8_header_parser.h" |
Henrik Kjellander | 98f5351 | 2015-10-28 18:17:40 +0100 | [diff] [blame] | 34 | #include "webrtc/system_wrappers/include/field_trial.h" |
| 35 | #include "webrtc/system_wrappers/include/logcat_trace_context.h" |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 36 | |
| 37 | using rtc::Bind; |
| 38 | using rtc::Thread; |
| 39 | using rtc::ThreadManager; |
| 40 | using rtc::scoped_ptr; |
| 41 | |
| 42 | using webrtc::CodecSpecificInfo; |
| 43 | using webrtc::EncodedImage; |
Miguel Casas-Sanchez | 4765070 | 2015-05-29 17:21:40 -0700 | [diff] [blame] | 44 | using webrtc::VideoFrame; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 45 | using webrtc::RTPFragmentationHeader; |
| 46 | using webrtc::VideoCodec; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 47 | using webrtc::VideoCodecType; |
| 48 | using webrtc::kVideoCodecH264; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 49 | using webrtc::kVideoCodecVP8; |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 50 | using webrtc::kVideoCodecVP9; |
Alex Glaznev | a9d0892 | 2016-02-19 15:24:06 -0800 | [diff] [blame] | 51 | using webrtc::QualityScaler; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 52 | |
| 53 | namespace webrtc_jni { |
| 54 | |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 55 | // H.264 start code length. |
| 56 | #define H264_SC_LENGTH 4 |
| 57 | // Maximum allowed NALUs in one output frame. |
| 58 | #define MAX_NALUS_PERFRAME 32 |
| 59 | // Maximum supported HW video encoder resolution. |
| 60 | #define MAX_VIDEO_WIDTH 1280 |
| 61 | #define MAX_VIDEO_HEIGHT 1280 |
| 62 | // Maximum supported HW video encoder fps. |
| 63 | #define MAX_VIDEO_FPS 30 |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 64 | // Maximum allowed fps value in SetRates() call. |
| 65 | #define MAX_ALLOWED_VIDEO_FPS 60 |
| 66 | // Maximum allowed frames in encoder input queue. |
| 67 | #define MAX_ENCODER_Q_SIZE 2 |
glaznev | 919ff75 | 2016-01-27 15:01:03 -0800 | [diff] [blame] | 68 | // Maximum amount of dropped frames caused by full encoder queue - exceeding |
| 69 | // this threshold means that encoder probably got stuck and need to be reset. |
| 70 | #define ENCODER_STALL_FRAMEDROP_THRESHOLD 60 |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 71 | |
| 72 | // Logging macros. |
| 73 | #define TAG_ENCODER "MediaCodecVideoEncoder" |
| 74 | #ifdef TRACK_BUFFER_TIMING |
| 75 | #define ALOGV(...) |
| 76 | __android_log_print(ANDROID_LOG_VERBOSE, TAG_ENCODER, __VA_ARGS__) |
| 77 | #else |
| 78 | #define ALOGV(...) |
| 79 | #endif |
| 80 | #define ALOGD LOG_TAG(rtc::LS_INFO, TAG_ENCODER) |
| 81 | #define ALOGW LOG_TAG(rtc::LS_WARNING, TAG_ENCODER) |
| 82 | #define ALOGE LOG_TAG(rtc::LS_ERROR, TAG_ENCODER) |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 83 | |
asapersson | 1d61a51 | 2016-01-20 01:13:46 -0800 | [diff] [blame] | 84 | namespace { |
| 85 | // Maximum time limit between incoming frames before requesting a key frame. |
| 86 | const size_t kFrameDiffThresholdMs = 1100; |
| 87 | const int kMinKeyFrameInterval = 2; |
| 88 | } // namespace |
| 89 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 90 | // MediaCodecVideoEncoder is a webrtc::VideoEncoder implementation that uses |
| 91 | // Android's MediaCodec SDK API behind the scenes to implement (hopefully) |
| 92 | // HW-backed video encode. This C++ class is implemented as a very thin shim, |
| 93 | // delegating all of the interesting work to org.webrtc.MediaCodecVideoEncoder. |
| 94 | // MediaCodecVideoEncoder is created, operated, and destroyed on a single |
| 95 | // thread, currently the libjingle Worker thread. |
| 96 | class MediaCodecVideoEncoder : public webrtc::VideoEncoder, |
| 97 | public rtc::MessageHandler { |
| 98 | public: |
| 99 | virtual ~MediaCodecVideoEncoder(); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 100 | MediaCodecVideoEncoder(JNIEnv* jni, |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 101 | VideoCodecType codecType, |
| 102 | jobject egl_context); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 103 | |
| 104 | // webrtc::VideoEncoder implementation. Everything trampolines to |
| 105 | // |codec_thread_| for execution. |
| 106 | int32_t InitEncode(const webrtc::VideoCodec* codec_settings, |
| 107 | int32_t /* number_of_cores */, |
| 108 | size_t /* max_payload_size */) override; |
pbos | 22993e1 | 2015-10-19 02:39:06 -0700 | [diff] [blame] | 109 | int32_t Encode(const webrtc::VideoFrame& input_image, |
| 110 | const webrtc::CodecSpecificInfo* /* codec_specific_info */, |
| 111 | const std::vector<webrtc::FrameType>* frame_types) override; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 112 | int32_t RegisterEncodeCompleteCallback( |
| 113 | webrtc::EncodedImageCallback* callback) override; |
| 114 | int32_t Release() override; |
| 115 | int32_t SetChannelParameters(uint32_t /* packet_loss */, |
| 116 | int64_t /* rtt */) override; |
| 117 | int32_t SetRates(uint32_t new_bit_rate, uint32_t frame_rate) override; |
| 118 | |
| 119 | // rtc::MessageHandler implementation. |
| 120 | void OnMessage(rtc::Message* msg) override; |
| 121 | |
jackychen | 61b4d51 | 2015-04-21 15:30:11 -0700 | [diff] [blame] | 122 | void OnDroppedFrame() override; |
| 123 | |
jackychen | 6e2ce6e | 2015-07-13 16:26:33 -0700 | [diff] [blame] | 124 | int GetTargetFramerate() override; |
| 125 | |
Per | ec2922f | 2016-01-27 15:25:46 +0100 | [diff] [blame] | 126 | bool SupportsNativeHandle() const override { return egl_context_ != nullptr; } |
Peter Boström | b7d9a97 | 2015-12-18 16:01:11 +0100 | [diff] [blame] | 127 | const char* ImplementationName() const override; |
| 128 | |
| 129 | private: |
| 130 | // CHECK-fail if not running on |codec_thread_|. |
| 131 | void CheckOnCodecThread(); |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 132 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 133 | private: |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 134 | // ResetCodecOnCodecThread() calls ReleaseOnCodecThread() and |
| 135 | // InitEncodeOnCodecThread() in an attempt to restore the codec to an |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 136 | // operable state. Necessary after all manner of OMX-layer errors. |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 137 | bool ResetCodecOnCodecThread(); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 138 | |
| 139 | // Implementation of webrtc::VideoEncoder methods above, all running on the |
| 140 | // codec thread exclusively. |
| 141 | // |
| 142 | // If width==0 then this is assumed to be a re-initialization and the |
| 143 | // previously-current values are reused instead of the passed parameters |
| 144 | // (makes it easier to reason about thread-safety). |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 145 | int32_t InitEncodeOnCodecThread(int width, int height, int kbps, int fps, |
| 146 | bool use_surface); |
| 147 | // Reconfigure to match |frame| in width, height. Also reconfigures the |
| 148 | // encoder if |frame| is a texture/byte buffer and the encoder is initialized |
| 149 | // for byte buffer/texture. Returns false if reconfiguring fails. |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 150 | bool MaybeReconfigureEncoderOnCodecThread(const webrtc::VideoFrame& frame); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 151 | int32_t EncodeOnCodecThread( |
Miguel Casas-Sanchez | 4765070 | 2015-05-29 17:21:40 -0700 | [diff] [blame] | 152 | const webrtc::VideoFrame& input_image, |
pbos | 22993e1 | 2015-10-19 02:39:06 -0700 | [diff] [blame] | 153 | const std::vector<webrtc::FrameType>* frame_types); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 154 | bool EncodeByteBufferOnCodecThread(JNIEnv* jni, |
| 155 | bool key_frame, const webrtc::VideoFrame& frame, int input_buffer_index); |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 156 | bool EncodeTextureOnCodecThread(JNIEnv* jni, |
| 157 | bool key_frame, const webrtc::VideoFrame& frame); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 158 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 159 | int32_t RegisterEncodeCompleteCallbackOnCodecThread( |
| 160 | webrtc::EncodedImageCallback* callback); |
| 161 | int32_t ReleaseOnCodecThread(); |
| 162 | int32_t SetRatesOnCodecThread(uint32_t new_bit_rate, uint32_t frame_rate); |
| 163 | |
| 164 | // Helper accessors for MediaCodecVideoEncoder$OutputBufferInfo members. |
| 165 | int GetOutputBufferInfoIndex(JNIEnv* jni, jobject j_output_buffer_info); |
| 166 | jobject GetOutputBufferInfoBuffer(JNIEnv* jni, jobject j_output_buffer_info); |
| 167 | bool GetOutputBufferInfoIsKeyFrame(JNIEnv* jni, jobject j_output_buffer_info); |
| 168 | jlong GetOutputBufferInfoPresentationTimestampUs( |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 169 | JNIEnv* jni, jobject j_output_buffer_info); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 170 | |
| 171 | // Deliver any outputs pending in the MediaCodec to our |callback_| and return |
| 172 | // true on success. |
| 173 | bool DeliverPendingOutputs(JNIEnv* jni); |
| 174 | |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 175 | // Search for H.264 start codes. |
| 176 | int32_t NextNaluPosition(uint8_t *buffer, size_t buffer_size); |
| 177 | |
glaznev | 9429148 | 2016-02-01 13:17:18 -0800 | [diff] [blame] | 178 | // Displays encoder statistics. |
| 179 | void LogStatistics(bool force_log); |
| 180 | |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 181 | // Type of video codec. |
| 182 | VideoCodecType codecType_; |
| 183 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 184 | // Valid all the time since RegisterEncodeCompleteCallback() Invoke()s to |
| 185 | // |codec_thread_| synchronously. |
| 186 | webrtc::EncodedImageCallback* callback_; |
| 187 | |
| 188 | // State that is constant for the lifetime of this object once the ctor |
| 189 | // returns. |
| 190 | scoped_ptr<Thread> codec_thread_; // Thread on which to operate MediaCodec. |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 191 | rtc::ThreadChecker codec_thread_checker_; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 192 | ScopedGlobalRef<jclass> j_media_codec_video_encoder_class_; |
| 193 | ScopedGlobalRef<jobject> j_media_codec_video_encoder_; |
| 194 | jmethodID j_init_encode_method_; |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 195 | jmethodID j_get_input_buffers_method_; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 196 | jmethodID j_dequeue_input_buffer_method_; |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 197 | jmethodID j_encode_buffer_method_; |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 198 | jmethodID j_encode_texture_method_; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 199 | jmethodID j_release_method_; |
| 200 | jmethodID j_set_rates_method_; |
| 201 | jmethodID j_dequeue_output_buffer_method_; |
| 202 | jmethodID j_release_output_buffer_method_; |
| 203 | jfieldID j_color_format_field_; |
| 204 | jfieldID j_info_index_field_; |
| 205 | jfieldID j_info_buffer_field_; |
| 206 | jfieldID j_info_is_key_frame_field_; |
| 207 | jfieldID j_info_presentation_timestamp_us_field_; |
| 208 | |
| 209 | // State that is valid only between InitEncode() and the next Release(). |
| 210 | // Touched only on codec_thread_ so no explicit synchronization necessary. |
| 211 | int width_; // Frame width in pixels. |
| 212 | int height_; // Frame height in pixels. |
| 213 | bool inited_; |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 214 | bool use_surface_; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 215 | uint16_t picture_id_; |
| 216 | enum libyuv::FourCC encoder_fourcc_; // Encoder color space format. |
| 217 | int last_set_bitrate_kbps_; // Last-requested bitrate in kbps. |
| 218 | int last_set_fps_; // Last-requested frame rate. |
| 219 | int64_t current_timestamp_us_; // Current frame timestamps in us. |
| 220 | int frames_received_; // Number of frames received by encoder. |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 221 | int frames_encoded_; // Number of frames encoded by encoder. |
glaznev | 919ff75 | 2016-01-27 15:01:03 -0800 | [diff] [blame] | 222 | int frames_dropped_media_encoder_; // Number of frames dropped by encoder. |
| 223 | // Number of dropped frames caused by full queue. |
| 224 | int consecutive_full_queue_frame_drops_; |
glaznev | 9429148 | 2016-02-01 13:17:18 -0800 | [diff] [blame] | 225 | int64_t stat_start_time_ms_; // Start time for statistics. |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 226 | int current_frames_; // Number of frames in the current statistics interval. |
| 227 | int current_bytes_; // Encoded bytes in the current statistics interval. |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 228 | int current_acc_qp_; // Accumulated QP in the current statistics interval. |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 229 | int current_encoding_time_ms_; // Overall encoding time in the current second |
| 230 | int64_t last_input_timestamp_ms_; // Timestamp of last received yuv frame. |
| 231 | int64_t last_output_timestamp_ms_; // Timestamp of last encoded frame. |
Per | ba7dc72 | 2016-04-19 15:01:23 +0200 | [diff] [blame] | 232 | |
| 233 | struct InputFrameInfo { |
| 234 | InputFrameInfo(int64_t encode_start_time, |
| 235 | int32_t frame_timestamp, |
| 236 | int64_t frame_render_time_ms, |
| 237 | webrtc::VideoRotation rotation) |
| 238 | : encode_start_time(encode_start_time), |
| 239 | frame_timestamp(frame_timestamp), |
| 240 | frame_render_time_ms(frame_render_time_ms), |
| 241 | rotation(rotation) {} |
| 242 | // Time when video frame is sent to encoder input. |
| 243 | const int64_t encode_start_time; |
| 244 | |
| 245 | // Input frame information. |
| 246 | const int32_t frame_timestamp; |
| 247 | const int64_t frame_render_time_ms; |
| 248 | const webrtc::VideoRotation rotation; |
| 249 | }; |
| 250 | std::list<InputFrameInfo> input_frame_infos_; |
| 251 | int32_t output_timestamp_; // Last output frame timestamp from |
| 252 | // |input_frame_infos_|. |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 253 | int64_t output_render_time_ms_; // Last output frame render time from |
Per | ba7dc72 | 2016-04-19 15:01:23 +0200 | [diff] [blame] | 254 | // |input_frame_infos_|. |
| 255 | webrtc::VideoRotation output_rotation_; // Last output frame rotation from |
| 256 | // |input_frame_infos_|. |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 257 | // Frame size in bytes fed to MediaCodec. |
| 258 | int yuv_size_; |
| 259 | // True only when between a callback_->Encoded() call return a positive value |
| 260 | // and the next Encode() call being ignored. |
| 261 | bool drop_next_input_frame_; |
| 262 | // Global references; must be deleted in Release(). |
| 263 | std::vector<jobject> input_buffers_; |
Alex Glaznev | a9d0892 | 2016-02-19 15:24:06 -0800 | [diff] [blame] | 264 | QualityScaler quality_scaler_; |
jackychen | 61b4d51 | 2015-04-21 15:30:11 -0700 | [diff] [blame] | 265 | // Dynamic resolution change, off by default. |
| 266 | bool scale_; |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 267 | |
| 268 | // H264 bitstream parser, used to extract QP from encoded bitstreams. |
| 269 | webrtc::H264BitstreamParser h264_bitstream_parser_; |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 270 | |
| 271 | // VP9 variables to populate codec specific structure. |
| 272 | webrtc::GofInfoVP9 gof_; // Contains each frame's temporal information for |
| 273 | // non-flexible VP9 mode. |
| 274 | uint8_t tl0_pic_idx_; |
| 275 | size_t gof_idx_; |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 276 | |
| 277 | // EGL context - owned by factory, should not be allocated/destroyed |
| 278 | // by MediaCodecVideoEncoder. |
| 279 | jobject egl_context_; |
asapersson | 1d61a51 | 2016-01-20 01:13:46 -0800 | [diff] [blame] | 280 | |
| 281 | // Temporary fix for VP8. |
| 282 | // Sends a key frame if frames are largely spaced apart (possibly |
| 283 | // corresponding to a large image change). |
| 284 | int64_t last_frame_received_ms_; |
| 285 | int frames_received_since_last_key_; |
| 286 | webrtc::VideoCodecMode codec_mode_; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 287 | }; |
| 288 | |
| 289 | MediaCodecVideoEncoder::~MediaCodecVideoEncoder() { |
| 290 | // Call Release() to ensure no more callbacks to us after we are deleted. |
| 291 | Release(); |
| 292 | } |
| 293 | |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 294 | MediaCodecVideoEncoder::MediaCodecVideoEncoder( |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 295 | JNIEnv* jni, VideoCodecType codecType, jobject egl_context) : |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 296 | codecType_(codecType), |
| 297 | callback_(NULL), |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 298 | codec_thread_(new Thread()), |
| 299 | j_media_codec_video_encoder_class_( |
| 300 | jni, |
| 301 | FindClass(jni, "org/webrtc/MediaCodecVideoEncoder")), |
| 302 | j_media_codec_video_encoder_( |
| 303 | jni, |
| 304 | jni->NewObject(*j_media_codec_video_encoder_class_, |
| 305 | GetMethodID(jni, |
| 306 | *j_media_codec_video_encoder_class_, |
| 307 | "<init>", |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 308 | "()V"))), |
kjellander | 60ca31b | 2016-01-04 10:15:53 -0800 | [diff] [blame] | 309 | inited_(false), |
| 310 | use_surface_(false), |
| 311 | picture_id_(0), |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 312 | egl_context_(egl_context) { |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 313 | ScopedLocalRefFrame local_ref_frame(jni); |
| 314 | // It would be nice to avoid spinning up a new thread per MediaCodec, and |
| 315 | // instead re-use e.g. the PeerConnectionFactory's |worker_thread_|, but bug |
| 316 | // 2732 means that deadlocks abound. This class synchronously trampolines |
| 317 | // to |codec_thread_|, so if anything else can be coming to _us_ from |
| 318 | // |codec_thread_|, or from any thread holding the |_sendCritSect| described |
| 319 | // in the bug, we have a problem. For now work around that with a dedicated |
| 320 | // thread. |
| 321 | codec_thread_->SetName("MediaCodecVideoEncoder", NULL); |
henrikg | 91d6ede | 2015-09-17 00:24:34 -0700 | [diff] [blame] | 322 | RTC_CHECK(codec_thread_->Start()) << "Failed to start MediaCodecVideoEncoder"; |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 323 | codec_thread_checker_.DetachFromThread(); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 324 | jclass j_output_buffer_info_class = |
| 325 | FindClass(jni, "org/webrtc/MediaCodecVideoEncoder$OutputBufferInfo"); |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 326 | j_init_encode_method_ = GetMethodID( |
| 327 | jni, |
| 328 | *j_media_codec_video_encoder_class_, |
| 329 | "initEncode", |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 330 | "(Lorg/webrtc/MediaCodecVideoEncoder$VideoCodecType;" |
perkj | 48477c1 | 2015-12-18 00:34:37 -0800 | [diff] [blame] | 331 | "IIIILorg/webrtc/EglBase14$Context;)Z"); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 332 | j_get_input_buffers_method_ = GetMethodID( |
| 333 | jni, |
| 334 | *j_media_codec_video_encoder_class_, |
| 335 | "getInputBuffers", |
| 336 | "()[Ljava/nio/ByteBuffer;"); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 337 | j_dequeue_input_buffer_method_ = GetMethodID( |
| 338 | jni, *j_media_codec_video_encoder_class_, "dequeueInputBuffer", "()I"); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 339 | j_encode_buffer_method_ = GetMethodID( |
| 340 | jni, *j_media_codec_video_encoder_class_, "encodeBuffer", "(ZIIJ)Z"); |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 341 | j_encode_texture_method_ = GetMethodID( |
| 342 | jni, *j_media_codec_video_encoder_class_, "encodeTexture", |
| 343 | "(ZI[FJ)Z"); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 344 | j_release_method_ = |
| 345 | GetMethodID(jni, *j_media_codec_video_encoder_class_, "release", "()V"); |
| 346 | j_set_rates_method_ = GetMethodID( |
| 347 | jni, *j_media_codec_video_encoder_class_, "setRates", "(II)Z"); |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 348 | j_dequeue_output_buffer_method_ = GetMethodID( |
| 349 | jni, |
| 350 | *j_media_codec_video_encoder_class_, |
| 351 | "dequeueOutputBuffer", |
| 352 | "()Lorg/webrtc/MediaCodecVideoEncoder$OutputBufferInfo;"); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 353 | j_release_output_buffer_method_ = GetMethodID( |
| 354 | jni, *j_media_codec_video_encoder_class_, "releaseOutputBuffer", "(I)Z"); |
| 355 | |
| 356 | j_color_format_field_ = |
| 357 | GetFieldID(jni, *j_media_codec_video_encoder_class_, "colorFormat", "I"); |
| 358 | j_info_index_field_ = |
| 359 | GetFieldID(jni, j_output_buffer_info_class, "index", "I"); |
| 360 | j_info_buffer_field_ = GetFieldID( |
| 361 | jni, j_output_buffer_info_class, "buffer", "Ljava/nio/ByteBuffer;"); |
| 362 | j_info_is_key_frame_field_ = |
| 363 | GetFieldID(jni, j_output_buffer_info_class, "isKeyFrame", "Z"); |
| 364 | j_info_presentation_timestamp_us_field_ = GetFieldID( |
| 365 | jni, j_output_buffer_info_class, "presentationTimestampUs", "J"); |
| 366 | CHECK_EXCEPTION(jni) << "MediaCodecVideoEncoder ctor failed"; |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 367 | srand(time(NULL)); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 368 | AllowBlockingCalls(); |
| 369 | } |
| 370 | |
| 371 | int32_t MediaCodecVideoEncoder::InitEncode( |
| 372 | const webrtc::VideoCodec* codec_settings, |
| 373 | int32_t /* number_of_cores */, |
| 374 | size_t /* max_payload_size */) { |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 375 | if (codec_settings == NULL) { |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 376 | ALOGE << "NULL VideoCodec instance"; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 377 | return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 378 | } |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 379 | // Factory should guard against other codecs being used with us. |
henrikg | 91d6ede | 2015-09-17 00:24:34 -0700 | [diff] [blame] | 380 | RTC_CHECK(codec_settings->codecType == codecType_) |
| 381 | << "Unsupported codec " << codec_settings->codecType << " for " |
| 382 | << codecType_; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 383 | |
asapersson | 1d61a51 | 2016-01-20 01:13:46 -0800 | [diff] [blame] | 384 | codec_mode_ = codec_settings->mode; |
Alex Glaznev | a9d0892 | 2016-02-19 15:24:06 -0800 | [diff] [blame] | 385 | int init_width = codec_settings->width; |
| 386 | int init_height = codec_settings->height; |
Peter Boström | 7ace488 | 2016-04-14 00:54:56 +0200 | [diff] [blame] | 387 | scale_ = codecType_ != kVideoCodecVP9; |
Alex Glaznev | a9d0892 | 2016-02-19 15:24:06 -0800 | [diff] [blame] | 388 | |
| 389 | ALOGD << "InitEncode request: " << init_width << " x " << init_height; |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 390 | ALOGD << "Encoder automatic resize " << (scale_ ? "enabled" : "disabled"); |
Alex Glaznev | a9d0892 | 2016-02-19 15:24:06 -0800 | [diff] [blame] | 391 | |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 392 | if (scale_) { |
| 393 | if (codecType_ == kVideoCodecVP8) { |
| 394 | // QP is obtained from VP8-bitstream for HW, so the QP corresponds to the |
| 395 | // (internal) range: [0, 127]. And we cannot change QP_max in HW, so it is |
| 396 | // always = 127. Note that in SW, QP is that of the user-level range [0, |
| 397 | // 63]. |
Alex Glaznev | 79299af | 2016-04-12 16:39:39 -0700 | [diff] [blame] | 398 | const int kLowQpThreshold = 29; |
Peter Boström | 2c8a296 | 2016-04-18 12:58:02 +0200 | [diff] [blame] | 399 | const int kBadQpThreshold = 100; |
Alex Glaznev | a9d0892 | 2016-02-19 15:24:06 -0800 | [diff] [blame] | 400 | quality_scaler_.Init(kLowQpThreshold, kBadQpThreshold, false, |
pbos | cbac40d | 2016-04-13 02:51:02 -0700 | [diff] [blame] | 401 | codec_settings->startBitrate, codec_settings->width, |
| 402 | codec_settings->height, |
| 403 | codec_settings->maxFramerate); |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 404 | } else if (codecType_ == kVideoCodecH264) { |
| 405 | // H264 QP is in the range [0, 51]. |
Peter Boström | 2c8a296 | 2016-04-18 12:58:02 +0200 | [diff] [blame] | 406 | const int kLowQpThreshold = 24; |
| 407 | const int kBadQpThreshold = 39; |
Alex Glaznev | a9d0892 | 2016-02-19 15:24:06 -0800 | [diff] [blame] | 408 | quality_scaler_.Init(kLowQpThreshold, kBadQpThreshold, false, |
pbos | cbac40d | 2016-04-13 02:51:02 -0700 | [diff] [blame] | 409 | codec_settings->startBitrate, codec_settings->width, |
| 410 | codec_settings->height, |
| 411 | codec_settings->maxFramerate); |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 412 | } else { |
| 413 | // When adding codec support to additional hardware codecs, also configure |
| 414 | // their QP thresholds for scaling. |
| 415 | RTC_NOTREACHED() << "Unsupported codec without configured QP thresholds."; |
Alex Glaznev | a9d0892 | 2016-02-19 15:24:06 -0800 | [diff] [blame] | 416 | scale_ = false; |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 417 | } |
Alex Glaznev | a9d0892 | 2016-02-19 15:24:06 -0800 | [diff] [blame] | 418 | QualityScaler::Resolution res = quality_scaler_.GetScaledResolution(); |
Peter Boström | 926dfcd | 2016-04-14 14:48:10 +0200 | [diff] [blame] | 419 | init_width = res.width; |
| 420 | init_height = res.height; |
Alex Glaznev | a9d0892 | 2016-02-19 15:24:06 -0800 | [diff] [blame] | 421 | ALOGD << "Scaled resolution: " << init_width << " x " << init_height; |
jackychen | 61b4d51 | 2015-04-21 15:30:11 -0700 | [diff] [blame] | 422 | } |
Alex Glaznev | a9d0892 | 2016-02-19 15:24:06 -0800 | [diff] [blame] | 423 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 424 | return codec_thread_->Invoke<int32_t>( |
| 425 | Bind(&MediaCodecVideoEncoder::InitEncodeOnCodecThread, |
| 426 | this, |
Alex Glaznev | a9d0892 | 2016-02-19 15:24:06 -0800 | [diff] [blame] | 427 | init_width, |
| 428 | init_height, |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 429 | codec_settings->startBitrate, |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 430 | codec_settings->maxFramerate, |
| 431 | false /* use_surface */)); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 432 | } |
| 433 | |
| 434 | int32_t MediaCodecVideoEncoder::Encode( |
Miguel Casas-Sanchez | 4765070 | 2015-05-29 17:21:40 -0700 | [diff] [blame] | 435 | const webrtc::VideoFrame& frame, |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 436 | const webrtc::CodecSpecificInfo* /* codec_specific_info */, |
pbos | 22993e1 | 2015-10-19 02:39:06 -0700 | [diff] [blame] | 437 | const std::vector<webrtc::FrameType>* frame_types) { |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 438 | return codec_thread_->Invoke<int32_t>(Bind( |
| 439 | &MediaCodecVideoEncoder::EncodeOnCodecThread, this, frame, frame_types)); |
| 440 | } |
| 441 | |
| 442 | int32_t MediaCodecVideoEncoder::RegisterEncodeCompleteCallback( |
| 443 | webrtc::EncodedImageCallback* callback) { |
| 444 | return codec_thread_->Invoke<int32_t>( |
| 445 | Bind(&MediaCodecVideoEncoder::RegisterEncodeCompleteCallbackOnCodecThread, |
| 446 | this, |
| 447 | callback)); |
| 448 | } |
| 449 | |
| 450 | int32_t MediaCodecVideoEncoder::Release() { |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 451 | ALOGD << "EncoderRelease request"; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 452 | return codec_thread_->Invoke<int32_t>( |
| 453 | Bind(&MediaCodecVideoEncoder::ReleaseOnCodecThread, this)); |
| 454 | } |
| 455 | |
| 456 | int32_t MediaCodecVideoEncoder::SetChannelParameters(uint32_t /* packet_loss */, |
| 457 | int64_t /* rtt */) { |
| 458 | return WEBRTC_VIDEO_CODEC_OK; |
| 459 | } |
| 460 | |
| 461 | int32_t MediaCodecVideoEncoder::SetRates(uint32_t new_bit_rate, |
| 462 | uint32_t frame_rate) { |
| 463 | return codec_thread_->Invoke<int32_t>( |
| 464 | Bind(&MediaCodecVideoEncoder::SetRatesOnCodecThread, |
| 465 | this, |
| 466 | new_bit_rate, |
| 467 | frame_rate)); |
| 468 | } |
| 469 | |
| 470 | void MediaCodecVideoEncoder::OnMessage(rtc::Message* msg) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 471 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 472 | JNIEnv* jni = AttachCurrentThreadIfNeeded(); |
| 473 | ScopedLocalRefFrame local_ref_frame(jni); |
| 474 | |
| 475 | // We only ever send one message to |this| directly (not through a Bind()'d |
| 476 | // functor), so expect no ID/data. |
henrikg | 91d6ede | 2015-09-17 00:24:34 -0700 | [diff] [blame] | 477 | RTC_CHECK(!msg->message_id) << "Unexpected message!"; |
| 478 | RTC_CHECK(!msg->pdata) << "Unexpected message!"; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 479 | if (!inited_) { |
| 480 | return; |
| 481 | } |
| 482 | |
| 483 | // It would be nice to recover from a failure here if one happened, but it's |
| 484 | // unclear how to signal such a failure to the app, so instead we stay silent |
| 485 | // about it and let the next app-called API method reveal the borkedness. |
| 486 | DeliverPendingOutputs(jni); |
| 487 | codec_thread_->PostDelayed(kMediaCodecPollMs, this); |
| 488 | } |
| 489 | |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 490 | bool MediaCodecVideoEncoder::ResetCodecOnCodecThread() { |
| 491 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
| 492 | ALOGE << "ResetOnCodecThread"; |
| 493 | if (ReleaseOnCodecThread() != WEBRTC_VIDEO_CODEC_OK || |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 494 | InitEncodeOnCodecThread(width_, height_, 0, 0, false) != |
| 495 | WEBRTC_VIDEO_CODEC_OK) { |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 496 | // TODO(fischman): wouldn't it be nice if there was a way to gracefully |
| 497 | // degrade to a SW encoder at this point? There isn't one AFAICT :( |
| 498 | // https://code.google.com/p/webrtc/issues/detail?id=2920 |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 499 | return false; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 500 | } |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 501 | return true; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 502 | } |
| 503 | |
| 504 | int32_t MediaCodecVideoEncoder::InitEncodeOnCodecThread( |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 505 | int width, int height, int kbps, int fps, bool use_surface) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 506 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 507 | RTC_CHECK(!use_surface || egl_context_ != nullptr) << "EGL context not set."; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 508 | JNIEnv* jni = AttachCurrentThreadIfNeeded(); |
| 509 | ScopedLocalRefFrame local_ref_frame(jni); |
| 510 | |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 511 | ALOGD << "InitEncodeOnCodecThread Type: " << (int)codecType_ << ", " << |
| 512 | width << " x " << height << ". Bitrate: " << kbps << |
| 513 | " kbps. Fps: " << fps; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 514 | if (kbps == 0) { |
| 515 | kbps = last_set_bitrate_kbps_; |
| 516 | } |
| 517 | if (fps == 0) { |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 518 | fps = MAX_VIDEO_FPS; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 519 | } |
| 520 | |
| 521 | width_ = width; |
| 522 | height_ = height; |
| 523 | last_set_bitrate_kbps_ = kbps; |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 524 | last_set_fps_ = (fps < MAX_VIDEO_FPS) ? fps : MAX_VIDEO_FPS; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 525 | yuv_size_ = width_ * height_ * 3 / 2; |
| 526 | frames_received_ = 0; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 527 | frames_encoded_ = 0; |
glaznev | 919ff75 | 2016-01-27 15:01:03 -0800 | [diff] [blame] | 528 | frames_dropped_media_encoder_ = 0; |
| 529 | consecutive_full_queue_frame_drops_ = 0; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 530 | current_timestamp_us_ = 0; |
glaznev | 9429148 | 2016-02-01 13:17:18 -0800 | [diff] [blame] | 531 | stat_start_time_ms_ = GetCurrentTimeMs(); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 532 | current_frames_ = 0; |
| 533 | current_bytes_ = 0; |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 534 | current_acc_qp_ = 0; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 535 | current_encoding_time_ms_ = 0; |
| 536 | last_input_timestamp_ms_ = -1; |
| 537 | last_output_timestamp_ms_ = -1; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 538 | output_timestamp_ = 0; |
| 539 | output_render_time_ms_ = 0; |
Per | ba7dc72 | 2016-04-19 15:01:23 +0200 | [diff] [blame] | 540 | input_frame_infos_.clear(); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 541 | drop_next_input_frame_ = false; |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 542 | use_surface_ = use_surface; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 543 | picture_id_ = static_cast<uint16_t>(rand()) & 0x7FFF; |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 544 | gof_.SetGofInfoVP9(webrtc::TemporalStructureMode::kTemporalStructureMode1); |
| 545 | tl0_pic_idx_ = static_cast<uint8_t>(rand()); |
| 546 | gof_idx_ = 0; |
asapersson | 1d61a51 | 2016-01-20 01:13:46 -0800 | [diff] [blame] | 547 | last_frame_received_ms_ = -1; |
| 548 | frames_received_since_last_key_ = kMinKeyFrameInterval; |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 549 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 550 | // We enforce no extra stride/padding in the format creation step. |
Per | ec2922f | 2016-01-27 15:25:46 +0100 | [diff] [blame] | 551 | jobject j_video_codec_enum = JavaEnumFromIndexAndClassName( |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 552 | jni, "MediaCodecVideoEncoder$VideoCodecType", codecType_); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 553 | const bool encode_status = jni->CallBooleanMethod( |
| 554 | *j_media_codec_video_encoder_, j_init_encode_method_, |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 555 | j_video_codec_enum, width, height, kbps, fps, |
| 556 | (use_surface ? egl_context_ : nullptr)); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 557 | if (!encode_status) { |
| 558 | ALOGE << "Failed to configure encoder."; |
| 559 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 560 | } |
| 561 | CHECK_EXCEPTION(jni); |
| 562 | |
Per | 598242a | 2015-11-26 14:28:55 +0100 | [diff] [blame] | 563 | if (!use_surface) { |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 564 | jobjectArray input_buffers = reinterpret_cast<jobjectArray>( |
| 565 | jni->CallObjectMethod(*j_media_codec_video_encoder_, |
| 566 | j_get_input_buffers_method_)); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 567 | CHECK_EXCEPTION(jni); |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 568 | if (IsNull(jni, input_buffers)) { |
| 569 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 570 | } |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 571 | |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 572 | switch (GetIntField(jni, *j_media_codec_video_encoder_, |
| 573 | j_color_format_field_)) { |
| 574 | case COLOR_FormatYUV420Planar: |
| 575 | encoder_fourcc_ = libyuv::FOURCC_YU12; |
| 576 | break; |
| 577 | case COLOR_FormatYUV420SemiPlanar: |
| 578 | case COLOR_QCOM_FormatYUV420SemiPlanar: |
| 579 | case COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m: |
| 580 | encoder_fourcc_ = libyuv::FOURCC_NV12; |
| 581 | break; |
| 582 | default: |
| 583 | LOG(LS_ERROR) << "Wrong color format."; |
| 584 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 585 | } |
| 586 | size_t num_input_buffers = jni->GetArrayLength(input_buffers); |
| 587 | RTC_CHECK(input_buffers_.empty()) |
| 588 | << "Unexpected double InitEncode without Release"; |
| 589 | input_buffers_.resize(num_input_buffers); |
| 590 | for (size_t i = 0; i < num_input_buffers; ++i) { |
| 591 | input_buffers_[i] = |
| 592 | jni->NewGlobalRef(jni->GetObjectArrayElement(input_buffers, i)); |
| 593 | int64_t yuv_buffer_capacity = |
| 594 | jni->GetDirectBufferCapacity(input_buffers_[i]); |
| 595 | CHECK_EXCEPTION(jni); |
| 596 | RTC_CHECK(yuv_buffer_capacity >= yuv_size_) << "Insufficient capacity"; |
| 597 | } |
| 598 | } |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 599 | |
| 600 | inited_ = true; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 601 | codec_thread_->PostDelayed(kMediaCodecPollMs, this); |
| 602 | return WEBRTC_VIDEO_CODEC_OK; |
| 603 | } |
| 604 | |
| 605 | int32_t MediaCodecVideoEncoder::EncodeOnCodecThread( |
Miguel Casas-Sanchez | 4765070 | 2015-05-29 17:21:40 -0700 | [diff] [blame] | 606 | const webrtc::VideoFrame& frame, |
pbos | 22993e1 | 2015-10-19 02:39:06 -0700 | [diff] [blame] | 607 | const std::vector<webrtc::FrameType>* frame_types) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 608 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 609 | JNIEnv* jni = AttachCurrentThreadIfNeeded(); |
| 610 | ScopedLocalRefFrame local_ref_frame(jni); |
| 611 | |
| 612 | if (!inited_) { |
| 613 | return WEBRTC_VIDEO_CODEC_UNINITIALIZED; |
| 614 | } |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 615 | |
asapersson | 1d61a51 | 2016-01-20 01:13:46 -0800 | [diff] [blame] | 616 | bool send_key_frame = false; |
glaznev | 9429148 | 2016-02-01 13:17:18 -0800 | [diff] [blame] | 617 | if (codec_mode_ == webrtc::kRealtimeVideo) { |
asapersson | 1d61a51 | 2016-01-20 01:13:46 -0800 | [diff] [blame] | 618 | ++frames_received_since_last_key_; |
| 619 | int64_t now_ms = GetCurrentTimeMs(); |
| 620 | if (last_frame_received_ms_ != -1 && |
| 621 | (now_ms - last_frame_received_ms_) > kFrameDiffThresholdMs) { |
| 622 | // Add limit to prevent triggering a key for every frame for very low |
| 623 | // framerates (e.g. if frame diff > kFrameDiffThresholdMs). |
| 624 | if (frames_received_since_last_key_ > kMinKeyFrameInterval) { |
| 625 | ALOGD << "Send key, frame diff: " << (now_ms - last_frame_received_ms_); |
| 626 | send_key_frame = true; |
| 627 | } |
| 628 | frames_received_since_last_key_ = 0; |
| 629 | } |
| 630 | last_frame_received_ms_ = now_ms; |
| 631 | } |
| 632 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 633 | frames_received_++; |
| 634 | if (!DeliverPendingOutputs(jni)) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 635 | if (!ResetCodecOnCodecThread()) |
| 636 | return WEBRTC_VIDEO_CODEC_ERROR; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 637 | } |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 638 | if (frames_encoded_ < kMaxEncodedLogFrames) { |
Per | ba7dc72 | 2016-04-19 15:01:23 +0200 | [diff] [blame] | 639 | ALOGD << "Encoder frame in # " << (frames_received_ - 1) |
| 640 | << ". TS: " << (int)(current_timestamp_us_ / 1000) |
| 641 | << ". Q: " << input_frame_infos_.size() << ". Fps: " << last_set_fps_ |
| 642 | << ". Kbps: " << last_set_bitrate_kbps_; |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 643 | } |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 644 | |
| 645 | if (drop_next_input_frame_) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 646 | ALOGW << "Encoder drop frame - failed callback."; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 647 | drop_next_input_frame_ = false; |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 648 | current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_; |
glaznev | 919ff75 | 2016-01-27 15:01:03 -0800 | [diff] [blame] | 649 | frames_dropped_media_encoder_++; |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 650 | OnDroppedFrame(); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 651 | return WEBRTC_VIDEO_CODEC_OK; |
| 652 | } |
| 653 | |
henrikg | 91d6ede | 2015-09-17 00:24:34 -0700 | [diff] [blame] | 654 | RTC_CHECK(frame_types->size() == 1) << "Unexpected stream count"; |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 655 | |
Peter Boström | f7704d1 | 2016-04-11 16:42:40 +0200 | [diff] [blame] | 656 | // Check if we accumulated too many frames in encoder input buffers and drop |
| 657 | // frame if so. |
Per | ba7dc72 | 2016-04-19 15:01:23 +0200 | [diff] [blame] | 658 | if (input_frame_infos_.size() > MAX_ENCODER_Q_SIZE) { |
| 659 | ALOGD << "Already " << input_frame_infos_.size() |
| 660 | << " frames in the queue, dropping" |
Peter Boström | f7704d1 | 2016-04-11 16:42:40 +0200 | [diff] [blame] | 661 | << ". TS: " << (int)(current_timestamp_us_ / 1000) |
| 662 | << ". Fps: " << last_set_fps_ |
| 663 | << ". Consecutive drops: " << consecutive_full_queue_frame_drops_; |
| 664 | current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_; |
| 665 | consecutive_full_queue_frame_drops_++; |
| 666 | if (consecutive_full_queue_frame_drops_ >= |
| 667 | ENCODER_STALL_FRAMEDROP_THRESHOLD) { |
| 668 | ALOGE << "Encoder got stuck. Reset."; |
| 669 | ResetCodecOnCodecThread(); |
| 670 | return WEBRTC_VIDEO_CODEC_ERROR; |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 671 | } |
Peter Boström | f7704d1 | 2016-04-11 16:42:40 +0200 | [diff] [blame] | 672 | frames_dropped_media_encoder_++; |
| 673 | OnDroppedFrame(); |
| 674 | return WEBRTC_VIDEO_CODEC_OK; |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 675 | } |
glaznev | 919ff75 | 2016-01-27 15:01:03 -0800 | [diff] [blame] | 676 | consecutive_full_queue_frame_drops_ = 0; |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 677 | |
Per | 598242a | 2015-11-26 14:28:55 +0100 | [diff] [blame] | 678 | VideoFrame input_frame = frame; |
| 679 | if (scale_) { |
| 680 | // Check framerate before spatial resolution change. |
| 681 | quality_scaler_.OnEncodeFrame(frame); |
| 682 | const webrtc::QualityScaler::Resolution scaled_resolution = |
| 683 | quality_scaler_.GetScaledResolution(); |
| 684 | if (scaled_resolution.width != frame.width() || |
| 685 | scaled_resolution.height != frame.height()) { |
nisse | 26acec4 | 2016-04-15 03:43:39 -0700 | [diff] [blame] | 686 | if (frame.video_frame_buffer()->native_handle() != nullptr) { |
Per | 598242a | 2015-11-26 14:28:55 +0100 | [diff] [blame] | 687 | rtc::scoped_refptr<webrtc::VideoFrameBuffer> scaled_buffer( |
| 688 | static_cast<AndroidTextureBuffer*>( |
Per | 71f5a9a | 2015-12-11 09:32:37 +0100 | [diff] [blame] | 689 | frame.video_frame_buffer().get())->ScaleAndRotate( |
Per | 598242a | 2015-11-26 14:28:55 +0100 | [diff] [blame] | 690 | scaled_resolution.width, |
Per | 71f5a9a | 2015-12-11 09:32:37 +0100 | [diff] [blame] | 691 | scaled_resolution.height, |
| 692 | webrtc::kVideoRotation_0)); |
Per | 598242a | 2015-11-26 14:28:55 +0100 | [diff] [blame] | 693 | input_frame.set_video_frame_buffer(scaled_buffer); |
| 694 | } else { |
| 695 | input_frame = quality_scaler_.GetScaledFrame(frame); |
| 696 | } |
| 697 | } |
| 698 | } |
jackychen | 61b4d51 | 2015-04-21 15:30:11 -0700 | [diff] [blame] | 699 | |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 700 | if (!MaybeReconfigureEncoderOnCodecThread(input_frame)) { |
| 701 | ALOGE << "Failed to reconfigure encoder."; |
| 702 | return WEBRTC_VIDEO_CODEC_ERROR; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 703 | } |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 704 | |
Per | ba7dc72 | 2016-04-19 15:01:23 +0200 | [diff] [blame] | 705 | const int64_t time_before_calling_encode = GetCurrentTimeMs(); |
asapersson | 1d61a51 | 2016-01-20 01:13:46 -0800 | [diff] [blame] | 706 | const bool key_frame = |
| 707 | frame_types->front() != webrtc::kVideoFrameDelta || send_key_frame; |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 708 | bool encode_status = true; |
nisse | 26acec4 | 2016-04-15 03:43:39 -0700 | [diff] [blame] | 709 | if (!input_frame.video_frame_buffer()->native_handle()) { |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 710 | int j_input_buffer_index = jni->CallIntMethod(*j_media_codec_video_encoder_, |
| 711 | j_dequeue_input_buffer_method_); |
| 712 | CHECK_EXCEPTION(jni); |
| 713 | if (j_input_buffer_index == -1) { |
| 714 | // Video codec falls behind - no input buffer available. |
| 715 | ALOGW << "Encoder drop frame - no input buffers available"; |
glaznev | 919ff75 | 2016-01-27 15:01:03 -0800 | [diff] [blame] | 716 | current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_; |
| 717 | frames_dropped_media_encoder_++; |
| 718 | OnDroppedFrame(); |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 719 | return WEBRTC_VIDEO_CODEC_OK; // TODO(fischman): see webrtc bug 2887. |
| 720 | } |
| 721 | if (j_input_buffer_index == -2) { |
| 722 | ResetCodecOnCodecThread(); |
| 723 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 724 | } |
| 725 | encode_status = EncodeByteBufferOnCodecThread(jni, key_frame, input_frame, |
| 726 | j_input_buffer_index); |
| 727 | } else { |
| 728 | encode_status = EncodeTextureOnCodecThread(jni, key_frame, input_frame); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 729 | } |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 730 | |
| 731 | if (!encode_status) { |
| 732 | ALOGE << "Failed encode frame with timestamp: " << input_frame.timestamp(); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 733 | ResetCodecOnCodecThread(); |
perkj | 12f6802 | 2015-10-16 13:31:45 +0200 | [diff] [blame] | 734 | return WEBRTC_VIDEO_CODEC_ERROR; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 735 | } |
| 736 | |
Per | ba7dc72 | 2016-04-19 15:01:23 +0200 | [diff] [blame] | 737 | // Save input image timestamps for later output. |
| 738 | input_frame_infos_.emplace_back( |
| 739 | time_before_calling_encode, input_frame.timestamp(), |
| 740 | input_frame.render_time_ms(), input_frame.rotation()); |
| 741 | |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 742 | last_input_timestamp_ms_ = |
| 743 | current_timestamp_us_ / rtc::kNumMicrosecsPerMillisec; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 744 | |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 745 | current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_; |
| 746 | |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 747 | if (!DeliverPendingOutputs(jni)) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 748 | ALOGE << "Failed deliver pending outputs."; |
| 749 | ResetCodecOnCodecThread(); |
| 750 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 751 | } |
| 752 | return WEBRTC_VIDEO_CODEC_OK; |
| 753 | } |
| 754 | |
| 755 | bool MediaCodecVideoEncoder::MaybeReconfigureEncoderOnCodecThread( |
| 756 | const webrtc::VideoFrame& frame) { |
| 757 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
| 758 | |
nisse | 26acec4 | 2016-04-15 03:43:39 -0700 | [diff] [blame] | 759 | const bool is_texture_frame = |
| 760 | frame.video_frame_buffer()->native_handle() != nullptr; |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 761 | const bool reconfigure_due_to_format = is_texture_frame != use_surface_; |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 762 | const bool reconfigure_due_to_size = |
| 763 | frame.width() != width_ || frame.height() != height_; |
| 764 | |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 765 | if (reconfigure_due_to_format) { |
| 766 | ALOGD << "Reconfigure encoder due to format change. " |
| 767 | << (use_surface_ ? |
| 768 | "Reconfiguring to encode from byte buffer." : |
| 769 | "Reconfiguring to encode from texture."); |
glaznev | 9429148 | 2016-02-01 13:17:18 -0800 | [diff] [blame] | 770 | LogStatistics(true); |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 771 | } |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 772 | if (reconfigure_due_to_size) { |
glaznev | 9429148 | 2016-02-01 13:17:18 -0800 | [diff] [blame] | 773 | ALOGW << "Reconfigure encoder due to frame resolution change from " |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 774 | << width_ << " x " << height_ << " to " << frame.width() << " x " |
| 775 | << frame.height(); |
glaznev | 9429148 | 2016-02-01 13:17:18 -0800 | [diff] [blame] | 776 | LogStatistics(true); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 777 | width_ = frame.width(); |
| 778 | height_ = frame.height(); |
| 779 | } |
| 780 | |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 781 | if (!reconfigure_due_to_format && !reconfigure_due_to_size) |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 782 | return true; |
| 783 | |
| 784 | ReleaseOnCodecThread(); |
| 785 | |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 786 | return InitEncodeOnCodecThread(width_, height_, 0, 0 , is_texture_frame) == |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 787 | WEBRTC_VIDEO_CODEC_OK; |
| 788 | } |
| 789 | |
| 790 | bool MediaCodecVideoEncoder::EncodeByteBufferOnCodecThread(JNIEnv* jni, |
| 791 | bool key_frame, const webrtc::VideoFrame& frame, int input_buffer_index) { |
| 792 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 793 | RTC_CHECK(!use_surface_); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 794 | |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 795 | jobject j_input_buffer = input_buffers_[input_buffer_index]; |
| 796 | uint8_t* yuv_buffer = |
| 797 | reinterpret_cast<uint8_t*>(jni->GetDirectBufferAddress(j_input_buffer)); |
| 798 | CHECK_EXCEPTION(jni); |
| 799 | RTC_CHECK(yuv_buffer) << "Indirect buffer??"; |
| 800 | RTC_CHECK(!libyuv::ConvertFromI420( |
| 801 | frame.buffer(webrtc::kYPlane), frame.stride(webrtc::kYPlane), |
| 802 | frame.buffer(webrtc::kUPlane), frame.stride(webrtc::kUPlane), |
| 803 | frame.buffer(webrtc::kVPlane), frame.stride(webrtc::kVPlane), |
| 804 | yuv_buffer, width_, width_, height_, encoder_fourcc_)) |
| 805 | << "ConvertFromI420 failed"; |
| 806 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 807 | bool encode_status = jni->CallBooleanMethod(*j_media_codec_video_encoder_, |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 808 | j_encode_buffer_method_, |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 809 | key_frame, |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 810 | input_buffer_index, |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 811 | yuv_size_, |
| 812 | current_timestamp_us_); |
| 813 | CHECK_EXCEPTION(jni); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 814 | return encode_status; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 815 | } |
| 816 | |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 817 | bool MediaCodecVideoEncoder::EncodeTextureOnCodecThread(JNIEnv* jni, |
| 818 | bool key_frame, const webrtc::VideoFrame& frame) { |
| 819 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
| 820 | RTC_CHECK(use_surface_); |
nisse | 26acec4 | 2016-04-15 03:43:39 -0700 | [diff] [blame] | 821 | NativeHandleImpl* handle = static_cast<NativeHandleImpl*>( |
| 822 | frame.video_frame_buffer()->native_handle()); |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 823 | jfloatArray sampling_matrix = jni->NewFloatArray(16); |
| 824 | jni->SetFloatArrayRegion(sampling_matrix, 0, 16, handle->sampling_matrix); |
| 825 | |
| 826 | bool encode_status = jni->CallBooleanMethod(*j_media_codec_video_encoder_, |
| 827 | j_encode_texture_method_, |
| 828 | key_frame, |
| 829 | handle->oes_texture_id, |
| 830 | sampling_matrix, |
| 831 | current_timestamp_us_); |
| 832 | CHECK_EXCEPTION(jni); |
| 833 | return encode_status; |
| 834 | } |
| 835 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 836 | int32_t MediaCodecVideoEncoder::RegisterEncodeCompleteCallbackOnCodecThread( |
| 837 | webrtc::EncodedImageCallback* callback) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 838 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 839 | JNIEnv* jni = AttachCurrentThreadIfNeeded(); |
| 840 | ScopedLocalRefFrame local_ref_frame(jni); |
| 841 | callback_ = callback; |
| 842 | return WEBRTC_VIDEO_CODEC_OK; |
| 843 | } |
| 844 | |
| 845 | int32_t MediaCodecVideoEncoder::ReleaseOnCodecThread() { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 846 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 847 | if (!inited_) { |
| 848 | return WEBRTC_VIDEO_CODEC_OK; |
| 849 | } |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 850 | JNIEnv* jni = AttachCurrentThreadIfNeeded(); |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 851 | ALOGD << "EncoderReleaseOnCodecThread: Frames received: " << |
| 852 | frames_received_ << ". Encoded: " << frames_encoded_ << |
glaznev | 919ff75 | 2016-01-27 15:01:03 -0800 | [diff] [blame] | 853 | ". Dropped: " << frames_dropped_media_encoder_; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 854 | ScopedLocalRefFrame local_ref_frame(jni); |
| 855 | for (size_t i = 0; i < input_buffers_.size(); ++i) |
| 856 | jni->DeleteGlobalRef(input_buffers_[i]); |
| 857 | input_buffers_.clear(); |
| 858 | jni->CallVoidMethod(*j_media_codec_video_encoder_, j_release_method_); |
| 859 | CHECK_EXCEPTION(jni); |
| 860 | rtc::MessageQueueManager::Clear(this); |
| 861 | inited_ = false; |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 862 | use_surface_ = false; |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 863 | ALOGD << "EncoderReleaseOnCodecThread done."; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 864 | return WEBRTC_VIDEO_CODEC_OK; |
| 865 | } |
| 866 | |
| 867 | int32_t MediaCodecVideoEncoder::SetRatesOnCodecThread(uint32_t new_bit_rate, |
| 868 | uint32_t frame_rate) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 869 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 870 | frame_rate = (frame_rate < MAX_ALLOWED_VIDEO_FPS) ? |
| 871 | frame_rate : MAX_ALLOWED_VIDEO_FPS; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 872 | if (last_set_bitrate_kbps_ == new_bit_rate && |
| 873 | last_set_fps_ == frame_rate) { |
| 874 | return WEBRTC_VIDEO_CODEC_OK; |
| 875 | } |
glaznev | 919ff75 | 2016-01-27 15:01:03 -0800 | [diff] [blame] | 876 | if (scale_) { |
| 877 | quality_scaler_.ReportFramerate(frame_rate); |
| 878 | } |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 879 | JNIEnv* jni = AttachCurrentThreadIfNeeded(); |
| 880 | ScopedLocalRefFrame local_ref_frame(jni); |
| 881 | if (new_bit_rate > 0) { |
| 882 | last_set_bitrate_kbps_ = new_bit_rate; |
| 883 | } |
| 884 | if (frame_rate > 0) { |
| 885 | last_set_fps_ = frame_rate; |
| 886 | } |
| 887 | bool ret = jni->CallBooleanMethod(*j_media_codec_video_encoder_, |
| 888 | j_set_rates_method_, |
| 889 | last_set_bitrate_kbps_, |
| 890 | last_set_fps_); |
| 891 | CHECK_EXCEPTION(jni); |
| 892 | if (!ret) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 893 | ResetCodecOnCodecThread(); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 894 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 895 | } |
| 896 | return WEBRTC_VIDEO_CODEC_OK; |
| 897 | } |
| 898 | |
| 899 | int MediaCodecVideoEncoder::GetOutputBufferInfoIndex( |
| 900 | JNIEnv* jni, |
| 901 | jobject j_output_buffer_info) { |
| 902 | return GetIntField(jni, j_output_buffer_info, j_info_index_field_); |
| 903 | } |
| 904 | |
| 905 | jobject MediaCodecVideoEncoder::GetOutputBufferInfoBuffer( |
| 906 | JNIEnv* jni, |
| 907 | jobject j_output_buffer_info) { |
| 908 | return GetObjectField(jni, j_output_buffer_info, j_info_buffer_field_); |
| 909 | } |
| 910 | |
| 911 | bool MediaCodecVideoEncoder::GetOutputBufferInfoIsKeyFrame( |
| 912 | JNIEnv* jni, |
| 913 | jobject j_output_buffer_info) { |
| 914 | return GetBooleanField(jni, j_output_buffer_info, j_info_is_key_frame_field_); |
| 915 | } |
| 916 | |
| 917 | jlong MediaCodecVideoEncoder::GetOutputBufferInfoPresentationTimestampUs( |
| 918 | JNIEnv* jni, |
| 919 | jobject j_output_buffer_info) { |
| 920 | return GetLongField( |
| 921 | jni, j_output_buffer_info, j_info_presentation_timestamp_us_field_); |
| 922 | } |
| 923 | |
| 924 | bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 925 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 926 | while (true) { |
| 927 | jobject j_output_buffer_info = jni->CallObjectMethod( |
| 928 | *j_media_codec_video_encoder_, j_dequeue_output_buffer_method_); |
| 929 | CHECK_EXCEPTION(jni); |
| 930 | if (IsNull(jni, j_output_buffer_info)) { |
| 931 | break; |
| 932 | } |
| 933 | |
| 934 | int output_buffer_index = |
| 935 | GetOutputBufferInfoIndex(jni, j_output_buffer_info); |
| 936 | if (output_buffer_index == -1) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 937 | ResetCodecOnCodecThread(); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 938 | return false; |
| 939 | } |
| 940 | |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 941 | // Get key and config frame flags. |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 942 | jobject j_output_buffer = |
| 943 | GetOutputBufferInfoBuffer(jni, j_output_buffer_info); |
| 944 | bool key_frame = GetOutputBufferInfoIsKeyFrame(jni, j_output_buffer_info); |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 945 | |
| 946 | // Get frame timestamps from a queue - for non config frames only. |
| 947 | int64_t frame_encoding_time_ms = 0; |
| 948 | last_output_timestamp_ms_ = |
| 949 | GetOutputBufferInfoPresentationTimestampUs(jni, j_output_buffer_info) / |
| 950 | 1000; |
Per | ba7dc72 | 2016-04-19 15:01:23 +0200 | [diff] [blame] | 951 | if (!input_frame_infos_.empty()) { |
| 952 | const InputFrameInfo& frame_info = input_frame_infos_.front(); |
| 953 | output_timestamp_ = frame_info.frame_timestamp; |
| 954 | output_render_time_ms_ = frame_info.frame_render_time_ms; |
| 955 | output_rotation_ = frame_info.rotation; |
| 956 | frame_encoding_time_ms = |
| 957 | GetCurrentTimeMs() - frame_info.encode_start_time; |
| 958 | input_frame_infos_.pop_front(); |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 959 | } |
| 960 | |
| 961 | // Extract payload. |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 962 | size_t payload_size = jni->GetDirectBufferCapacity(j_output_buffer); |
Peter Boström | 0c4e06b | 2015-10-07 12:23:21 +0200 | [diff] [blame] | 963 | uint8_t* payload = reinterpret_cast<uint8_t*>( |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 964 | jni->GetDirectBufferAddress(j_output_buffer)); |
| 965 | CHECK_EXCEPTION(jni); |
| 966 | |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 967 | if (frames_encoded_ < kMaxEncodedLogFrames) { |
glaznev | 9429148 | 2016-02-01 13:17:18 -0800 | [diff] [blame] | 968 | int current_latency = |
| 969 | (int)(last_input_timestamp_ms_ - last_output_timestamp_ms_); |
| 970 | ALOGD << "Encoder frame out # " << frames_encoded_ << |
| 971 | ". Key: " << key_frame << |
| 972 | ". Size: " << payload_size << |
| 973 | ". TS: " << (int)last_output_timestamp_ms_ << |
| 974 | ". Latency: " << current_latency << |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 975 | ". EncTime: " << frame_encoding_time_ms; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 976 | } |
| 977 | |
| 978 | // Callback - return encoded frame. |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 979 | int32_t callback_status = 0; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 980 | if (callback_) { |
| 981 | scoped_ptr<webrtc::EncodedImage> image( |
| 982 | new webrtc::EncodedImage(payload, payload_size, payload_size)); |
| 983 | image->_encodedWidth = width_; |
| 984 | image->_encodedHeight = height_; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 985 | image->_timeStamp = output_timestamp_; |
| 986 | image->capture_time_ms_ = output_render_time_ms_; |
Per | ba7dc72 | 2016-04-19 15:01:23 +0200 | [diff] [blame] | 987 | image->rotation_ = output_rotation_; |
Peter Boström | 49e196a | 2015-10-23 15:58:18 +0200 | [diff] [blame] | 988 | image->_frameType = |
| 989 | (key_frame ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 990 | image->_completeFrame = true; |
asapersson | 075fb4b | 2015-10-29 08:49:14 -0700 | [diff] [blame] | 991 | image->adapt_reason_.quality_resolution_downscales = |
| 992 | scale_ ? quality_scaler_.downscale_shift() : -1; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 993 | |
| 994 | webrtc::CodecSpecificInfo info; |
| 995 | memset(&info, 0, sizeof(info)); |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 996 | info.codecType = codecType_; |
| 997 | if (codecType_ == kVideoCodecVP8) { |
| 998 | info.codecSpecific.VP8.pictureId = picture_id_; |
| 999 | info.codecSpecific.VP8.nonReference = false; |
| 1000 | info.codecSpecific.VP8.simulcastIdx = 0; |
| 1001 | info.codecSpecific.VP8.temporalIdx = webrtc::kNoTemporalIdx; |
| 1002 | info.codecSpecific.VP8.layerSync = false; |
| 1003 | info.codecSpecific.VP8.tl0PicIdx = webrtc::kNoTl0PicIdx; |
| 1004 | info.codecSpecific.VP8.keyIdx = webrtc::kNoKeyIdx; |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 1005 | } else if (codecType_ == kVideoCodecVP9) { |
| 1006 | if (key_frame) { |
| 1007 | gof_idx_ = 0; |
| 1008 | } |
| 1009 | info.codecSpecific.VP9.picture_id = picture_id_; |
| 1010 | info.codecSpecific.VP9.inter_pic_predicted = key_frame ? false : true; |
| 1011 | info.codecSpecific.VP9.flexible_mode = false; |
| 1012 | info.codecSpecific.VP9.ss_data_available = key_frame ? true : false; |
| 1013 | info.codecSpecific.VP9.tl0_pic_idx = tl0_pic_idx_++; |
| 1014 | info.codecSpecific.VP9.temporal_idx = webrtc::kNoTemporalIdx; |
| 1015 | info.codecSpecific.VP9.spatial_idx = webrtc::kNoSpatialIdx; |
| 1016 | info.codecSpecific.VP9.temporal_up_switch = true; |
| 1017 | info.codecSpecific.VP9.inter_layer_predicted = false; |
| 1018 | info.codecSpecific.VP9.gof_idx = |
| 1019 | static_cast<uint8_t>(gof_idx_++ % gof_.num_frames_in_gof); |
| 1020 | info.codecSpecific.VP9.num_spatial_layers = 1; |
| 1021 | info.codecSpecific.VP9.spatial_layer_resolution_present = false; |
| 1022 | if (info.codecSpecific.VP9.ss_data_available) { |
| 1023 | info.codecSpecific.VP9.spatial_layer_resolution_present = true; |
| 1024 | info.codecSpecific.VP9.width[0] = width_; |
| 1025 | info.codecSpecific.VP9.height[0] = height_; |
| 1026 | info.codecSpecific.VP9.gof.CopyGofInfoVP9(gof_); |
| 1027 | } |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1028 | } |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 1029 | picture_id_ = (picture_id_ + 1) & 0x7FFF; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1030 | |
| 1031 | // Generate a header describing a single fragment. |
| 1032 | webrtc::RTPFragmentationHeader header; |
| 1033 | memset(&header, 0, sizeof(header)); |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 1034 | if (codecType_ == kVideoCodecVP8 || codecType_ == kVideoCodecVP9) { |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1035 | header.VerifyAndAllocateFragmentationHeader(1); |
| 1036 | header.fragmentationOffset[0] = 0; |
| 1037 | header.fragmentationLength[0] = image->_length; |
| 1038 | header.fragmentationPlType[0] = 0; |
| 1039 | header.fragmentationTimeDiff[0] = 0; |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 1040 | if (codecType_ == kVideoCodecVP8 && scale_) { |
asapersson | 86b0160 | 2015-10-20 23:55:26 -0700 | [diff] [blame] | 1041 | int qp; |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 1042 | if (webrtc::vp8::GetQp(payload, payload_size, &qp)) { |
| 1043 | current_acc_qp_ += qp; |
asapersson | 86b0160 | 2015-10-20 23:55:26 -0700 | [diff] [blame] | 1044 | quality_scaler_.ReportQP(qp); |
asapersson | 24ebc44 | 2016-04-19 23:48:21 -0700 | [diff] [blame] | 1045 | image->qp_ = qp; |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 1046 | } |
asapersson | 86b0160 | 2015-10-20 23:55:26 -0700 | [diff] [blame] | 1047 | } |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1048 | } else if (codecType_ == kVideoCodecH264) { |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 1049 | if (scale_) { |
| 1050 | h264_bitstream_parser_.ParseBitstream(payload, payload_size); |
| 1051 | int qp; |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 1052 | if (h264_bitstream_parser_.GetLastSliceQp(&qp)) { |
| 1053 | current_acc_qp_ += qp; |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 1054 | quality_scaler_.ReportQP(qp); |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 1055 | } |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 1056 | } |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1057 | // For H.264 search for start codes. |
| 1058 | int32_t scPositions[MAX_NALUS_PERFRAME + 1] = {}; |
| 1059 | int32_t scPositionsLength = 0; |
| 1060 | int32_t scPosition = 0; |
| 1061 | while (scPositionsLength < MAX_NALUS_PERFRAME) { |
| 1062 | int32_t naluPosition = NextNaluPosition( |
| 1063 | payload + scPosition, payload_size - scPosition); |
| 1064 | if (naluPosition < 0) { |
| 1065 | break; |
| 1066 | } |
| 1067 | scPosition += naluPosition; |
| 1068 | scPositions[scPositionsLength++] = scPosition; |
| 1069 | scPosition += H264_SC_LENGTH; |
| 1070 | } |
| 1071 | if (scPositionsLength == 0) { |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 1072 | ALOGE << "Start code is not found!"; |
| 1073 | ALOGE << "Data:" << image->_buffer[0] << " " << image->_buffer[1] |
| 1074 | << " " << image->_buffer[2] << " " << image->_buffer[3] |
| 1075 | << " " << image->_buffer[4] << " " << image->_buffer[5]; |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 1076 | ResetCodecOnCodecThread(); |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1077 | return false; |
| 1078 | } |
| 1079 | scPositions[scPositionsLength] = payload_size; |
| 1080 | header.VerifyAndAllocateFragmentationHeader(scPositionsLength); |
| 1081 | for (size_t i = 0; i < scPositionsLength; i++) { |
| 1082 | header.fragmentationOffset[i] = scPositions[i] + H264_SC_LENGTH; |
| 1083 | header.fragmentationLength[i] = |
| 1084 | scPositions[i + 1] - header.fragmentationOffset[i]; |
| 1085 | header.fragmentationPlType[i] = 0; |
| 1086 | header.fragmentationTimeDiff[i] = 0; |
| 1087 | } |
| 1088 | } |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1089 | |
| 1090 | callback_status = callback_->Encoded(*image, &info, &header); |
| 1091 | } |
| 1092 | |
| 1093 | // Return output buffer back to the encoder. |
| 1094 | bool success = jni->CallBooleanMethod(*j_media_codec_video_encoder_, |
| 1095 | j_release_output_buffer_method_, |
| 1096 | output_buffer_index); |
| 1097 | CHECK_EXCEPTION(jni); |
| 1098 | if (!success) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 1099 | ResetCodecOnCodecThread(); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1100 | return false; |
| 1101 | } |
| 1102 | |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 1103 | // Calculate and print encoding statistics - every 3 seconds. |
| 1104 | frames_encoded_++; |
| 1105 | current_frames_++; |
| 1106 | current_bytes_ += payload_size; |
| 1107 | current_encoding_time_ms_ += frame_encoding_time_ms; |
glaznev | 9429148 | 2016-02-01 13:17:18 -0800 | [diff] [blame] | 1108 | LogStatistics(false); |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 1109 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1110 | if (callback_status > 0) { |
| 1111 | drop_next_input_frame_ = true; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1112 | // Theoretically could handle callback_status<0 here, but unclear what |
| 1113 | // that would mean for us. |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1114 | } |
| 1115 | } |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1116 | return true; |
| 1117 | } |
| 1118 | |
glaznev | 9429148 | 2016-02-01 13:17:18 -0800 | [diff] [blame] | 1119 | void MediaCodecVideoEncoder::LogStatistics(bool force_log) { |
| 1120 | int statistic_time_ms = GetCurrentTimeMs() - stat_start_time_ms_; |
| 1121 | if ((statistic_time_ms >= kMediaCodecStatisticsIntervalMs || force_log) && |
| 1122 | current_frames_ > 0 && statistic_time_ms > 0) { |
| 1123 | int current_bitrate = current_bytes_ * 8 / statistic_time_ms; |
| 1124 | int current_fps = |
| 1125 | (current_frames_ * 1000 + statistic_time_ms / 2) / statistic_time_ms; |
| 1126 | ALOGD << "Encoded frames: " << frames_encoded_ << |
| 1127 | ". Bitrate: " << current_bitrate << |
| 1128 | ", target: " << last_set_bitrate_kbps_ << " kbps" << |
| 1129 | ", fps: " << current_fps << |
| 1130 | ", encTime: " << (current_encoding_time_ms_ / current_frames_) << |
| 1131 | ". QP: " << (current_acc_qp_ / current_frames_) << |
| 1132 | " for last " << statistic_time_ms << " ms."; |
| 1133 | stat_start_time_ms_ = GetCurrentTimeMs(); |
| 1134 | current_frames_ = 0; |
| 1135 | current_bytes_ = 0; |
| 1136 | current_acc_qp_ = 0; |
| 1137 | current_encoding_time_ms_ = 0; |
| 1138 | } |
| 1139 | } |
| 1140 | |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1141 | int32_t MediaCodecVideoEncoder::NextNaluPosition( |
| 1142 | uint8_t *buffer, size_t buffer_size) { |
| 1143 | if (buffer_size < H264_SC_LENGTH) { |
| 1144 | return -1; |
| 1145 | } |
| 1146 | uint8_t *head = buffer; |
| 1147 | // Set end buffer pointer to 4 bytes before actual buffer end so we can |
| 1148 | // access head[1], head[2] and head[3] in a loop without buffer overrun. |
| 1149 | uint8_t *end = buffer + buffer_size - H264_SC_LENGTH; |
| 1150 | |
| 1151 | while (head < end) { |
| 1152 | if (head[0]) { |
| 1153 | head++; |
| 1154 | continue; |
| 1155 | } |
| 1156 | if (head[1]) { // got 00xx |
| 1157 | head += 2; |
| 1158 | continue; |
| 1159 | } |
| 1160 | if (head[2]) { // got 0000xx |
| 1161 | head += 3; |
| 1162 | continue; |
| 1163 | } |
| 1164 | if (head[3] != 0x01) { // got 000000xx |
glaznev@webrtc.org | dc08a23 | 2015-03-06 23:32:20 +0000 | [diff] [blame] | 1165 | head++; // xx != 1, continue searching. |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1166 | continue; |
| 1167 | } |
| 1168 | return (int32_t)(head - buffer); |
| 1169 | } |
| 1170 | return -1; |
| 1171 | } |
| 1172 | |
jackychen | 61b4d51 | 2015-04-21 15:30:11 -0700 | [diff] [blame] | 1173 | void MediaCodecVideoEncoder::OnDroppedFrame() { |
glaznev | f4decb5 | 2016-01-15 13:49:22 -0800 | [diff] [blame] | 1174 | // Report dropped frame to quality_scaler_. |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 1175 | if (scale_) |
| 1176 | quality_scaler_.ReportDroppedFrame(); |
jackychen | 61b4d51 | 2015-04-21 15:30:11 -0700 | [diff] [blame] | 1177 | } |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1178 | |
jackychen | 6e2ce6e | 2015-07-13 16:26:33 -0700 | [diff] [blame] | 1179 | int MediaCodecVideoEncoder::GetTargetFramerate() { |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 1180 | return scale_ ? quality_scaler_.GetTargetFramerate() : -1; |
jackychen | 6e2ce6e | 2015-07-13 16:26:33 -0700 | [diff] [blame] | 1181 | } |
| 1182 | |
Peter Boström | b7d9a97 | 2015-12-18 16:01:11 +0100 | [diff] [blame] | 1183 | const char* MediaCodecVideoEncoder::ImplementationName() const { |
| 1184 | return "MediaCodec"; |
| 1185 | } |
| 1186 | |
perkj | 461121c | 2016-02-15 06:28:36 -0800 | [diff] [blame] | 1187 | MediaCodecVideoEncoderFactory::MediaCodecVideoEncoderFactory() |
| 1188 | : egl_context_(nullptr) { |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1189 | JNIEnv* jni = AttachCurrentThreadIfNeeded(); |
| 1190 | ScopedLocalRefFrame local_ref_frame(jni); |
| 1191 | jclass j_encoder_class = FindClass(jni, "org/webrtc/MediaCodecVideoEncoder"); |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1192 | supported_codecs_.clear(); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1193 | |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1194 | bool is_vp8_hw_supported = jni->CallStaticBooleanMethod( |
| 1195 | j_encoder_class, |
| 1196 | GetStaticMethodID(jni, j_encoder_class, "isVp8HwSupported", "()Z")); |
| 1197 | CHECK_EXCEPTION(jni); |
| 1198 | if (is_vp8_hw_supported) { |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 1199 | ALOGD << "VP8 HW Encoder supported."; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1200 | supported_codecs_.push_back(VideoCodec(kVideoCodecVP8, "VP8", |
| 1201 | MAX_VIDEO_WIDTH, MAX_VIDEO_HEIGHT, MAX_VIDEO_FPS)); |
| 1202 | } |
| 1203 | |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 1204 | bool is_vp9_hw_supported = jni->CallStaticBooleanMethod( |
| 1205 | j_encoder_class, |
| 1206 | GetStaticMethodID(jni, j_encoder_class, "isVp9HwSupported", "()Z")); |
| 1207 | CHECK_EXCEPTION(jni); |
| 1208 | if (is_vp9_hw_supported) { |
| 1209 | ALOGD << "VP9 HW Encoder supported."; |
| 1210 | supported_codecs_.push_back(VideoCodec(kVideoCodecVP9, "VP9", |
| 1211 | MAX_VIDEO_WIDTH, MAX_VIDEO_HEIGHT, MAX_VIDEO_FPS)); |
| 1212 | } |
| 1213 | |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1214 | bool is_h264_hw_supported = jni->CallStaticBooleanMethod( |
| 1215 | j_encoder_class, |
| 1216 | GetStaticMethodID(jni, j_encoder_class, "isH264HwSupported", "()Z")); |
| 1217 | CHECK_EXCEPTION(jni); |
| 1218 | if (is_h264_hw_supported) { |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 1219 | ALOGD << "H.264 HW Encoder supported."; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1220 | supported_codecs_.push_back(VideoCodec(kVideoCodecH264, "H264", |
| 1221 | MAX_VIDEO_WIDTH, MAX_VIDEO_HEIGHT, MAX_VIDEO_FPS)); |
| 1222 | } |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1223 | } |
| 1224 | |
Per | ec2922f | 2016-01-27 15:25:46 +0100 | [diff] [blame] | 1225 | MediaCodecVideoEncoderFactory::~MediaCodecVideoEncoderFactory() { |
| 1226 | ALOGD << "MediaCodecVideoEncoderFactory dtor"; |
perkj | 461121c | 2016-02-15 06:28:36 -0800 | [diff] [blame] | 1227 | if (egl_context_) { |
| 1228 | JNIEnv* jni = AttachCurrentThreadIfNeeded(); |
| 1229 | jni->DeleteGlobalRef(egl_context_); |
| 1230 | } |
Per | ec2922f | 2016-01-27 15:25:46 +0100 | [diff] [blame] | 1231 | } |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1232 | |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 1233 | void MediaCodecVideoEncoderFactory::SetEGLContext( |
perkj | 461121c | 2016-02-15 06:28:36 -0800 | [diff] [blame] | 1234 | JNIEnv* jni, jobject egl_context) { |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 1235 | ALOGD << "MediaCodecVideoEncoderFactory::SetEGLContext"; |
Per | fd22e6c | 2016-02-18 11:35:48 +0100 | [diff] [blame] | 1236 | if (egl_context_) { |
| 1237 | jni->DeleteGlobalRef(egl_context_); |
| 1238 | egl_context_ = nullptr; |
| 1239 | } |
perkj | 461121c | 2016-02-15 06:28:36 -0800 | [diff] [blame] | 1240 | egl_context_ = jni->NewGlobalRef(egl_context); |
| 1241 | if (CheckException(jni)) { |
| 1242 | ALOGE << "error calling NewGlobalRef for EGL Context."; |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 1243 | } |
| 1244 | } |
| 1245 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1246 | webrtc::VideoEncoder* MediaCodecVideoEncoderFactory::CreateVideoEncoder( |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1247 | VideoCodecType type) { |
| 1248 | if (supported_codecs_.empty()) { |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 1249 | ALOGW << "No HW video encoder for type " << (int)type; |
Per | ec2922f | 2016-01-27 15:25:46 +0100 | [diff] [blame] | 1250 | return nullptr; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1251 | } |
| 1252 | for (std::vector<VideoCodec>::const_iterator it = supported_codecs_.begin(); |
| 1253 | it != supported_codecs_.end(); ++it) { |
| 1254 | if (it->type == type) { |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 1255 | ALOGD << "Create HW video encoder for type " << (int)type << |
| 1256 | " (" << it->name << ")."; |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 1257 | return new MediaCodecVideoEncoder(AttachCurrentThreadIfNeeded(), type, |
perkj | 461121c | 2016-02-15 06:28:36 -0800 | [diff] [blame] | 1258 | egl_context_); |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1259 | } |
| 1260 | } |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 1261 | ALOGW << "Can not find HW video encoder for type " << (int)type; |
Per | ec2922f | 2016-01-27 15:25:46 +0100 | [diff] [blame] | 1262 | return nullptr; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1263 | } |
| 1264 | |
| 1265 | const std::vector<MediaCodecVideoEncoderFactory::VideoCodec>& |
| 1266 | MediaCodecVideoEncoderFactory::codecs() const { |
| 1267 | return supported_codecs_; |
| 1268 | } |
| 1269 | |
| 1270 | void MediaCodecVideoEncoderFactory::DestroyVideoEncoder( |
| 1271 | webrtc::VideoEncoder* encoder) { |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 1272 | ALOGD << "Destroy video encoder."; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1273 | delete encoder; |
| 1274 | } |
| 1275 | |
| 1276 | } // namespace webrtc_jni |