glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1 | /* |
| 2 | * libjingle |
| 3 | * Copyright 2015 Google Inc. |
| 4 | * |
| 5 | * Redistribution and use in source and binary forms, with or without |
| 6 | * modification, are permitted provided that the following conditions are met: |
| 7 | * |
| 8 | * 1. Redistributions of source code must retain the above copyright notice, |
| 9 | * this list of conditions and the following disclaimer. |
| 10 | * 2. Redistributions in binary form must reproduce the above copyright notice, |
| 11 | * this list of conditions and the following disclaimer in the documentation |
| 12 | * and/or other materials provided with the distribution. |
| 13 | * 3. The name of the author may not be used to endorse or promote products |
| 14 | * derived from this software without specific prior written permission. |
| 15 | * |
| 16 | * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED |
| 17 | * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF |
| 18 | * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO |
| 19 | * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
| 20 | * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, |
| 21 | * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; |
| 22 | * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, |
| 23 | * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR |
| 24 | * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF |
| 25 | * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. |
| 26 | * |
| 27 | */ |
| 28 | |
| 29 | #include "talk/app/webrtc/java/jni/androidmediaencoder_jni.h" |
| 30 | #include "talk/app/webrtc/java/jni/classreferenceholder.h" |
| 31 | #include "talk/app/webrtc/java/jni/androidmediacodeccommon.h" |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 32 | #include "talk/app/webrtc/java/jni/native_handle_impl.h" |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 33 | #include "webrtc/base/bind.h" |
| 34 | #include "webrtc/base/checks.h" |
| 35 | #include "webrtc/base/logging.h" |
| 36 | #include "webrtc/base/thread.h" |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 37 | #include "webrtc/base/thread_checker.h" |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 38 | #include "webrtc/modules/rtp_rtcp/source/h264_bitstream_parser.h" |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 39 | #include "webrtc/modules/video_coding/include/video_codec_interface.h" |
kjellander@webrtc.org | b7ce964 | 2015-11-18 23:04:10 +0100 | [diff] [blame] | 40 | #include "webrtc/modules/video_coding/utility/quality_scaler.h" |
| 41 | #include "webrtc/modules/video_coding/utility/vp8_header_parser.h" |
Henrik Kjellander | 98f5351 | 2015-10-28 18:17:40 +0100 | [diff] [blame] | 42 | #include "webrtc/system_wrappers/include/field_trial.h" |
| 43 | #include "webrtc/system_wrappers/include/logcat_trace_context.h" |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 44 | #include "third_party/libyuv/include/libyuv/convert.h" |
| 45 | #include "third_party/libyuv/include/libyuv/convert_from.h" |
| 46 | #include "third_party/libyuv/include/libyuv/video_common.h" |
| 47 | |
| 48 | using rtc::Bind; |
| 49 | using rtc::Thread; |
| 50 | using rtc::ThreadManager; |
| 51 | using rtc::scoped_ptr; |
| 52 | |
| 53 | using webrtc::CodecSpecificInfo; |
| 54 | using webrtc::EncodedImage; |
Miguel Casas-Sanchez | 4765070 | 2015-05-29 17:21:40 -0700 | [diff] [blame] | 55 | using webrtc::VideoFrame; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 56 | using webrtc::RTPFragmentationHeader; |
| 57 | using webrtc::VideoCodec; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 58 | using webrtc::VideoCodecType; |
| 59 | using webrtc::kVideoCodecH264; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 60 | using webrtc::kVideoCodecVP8; |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 61 | using webrtc::kVideoCodecVP9; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 62 | |
| 63 | namespace webrtc_jni { |
| 64 | |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 65 | // H.264 start code length. |
| 66 | #define H264_SC_LENGTH 4 |
| 67 | // Maximum allowed NALUs in one output frame. |
| 68 | #define MAX_NALUS_PERFRAME 32 |
| 69 | // Maximum supported HW video encoder resolution. |
| 70 | #define MAX_VIDEO_WIDTH 1280 |
| 71 | #define MAX_VIDEO_HEIGHT 1280 |
| 72 | // Maximum supported HW video encoder fps. |
| 73 | #define MAX_VIDEO_FPS 30 |
| 74 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 75 | // MediaCodecVideoEncoder is a webrtc::VideoEncoder implementation that uses |
| 76 | // Android's MediaCodec SDK API behind the scenes to implement (hopefully) |
| 77 | // HW-backed video encode. This C++ class is implemented as a very thin shim, |
| 78 | // delegating all of the interesting work to org.webrtc.MediaCodecVideoEncoder. |
| 79 | // MediaCodecVideoEncoder is created, operated, and destroyed on a single |
| 80 | // thread, currently the libjingle Worker thread. |
| 81 | class MediaCodecVideoEncoder : public webrtc::VideoEncoder, |
| 82 | public rtc::MessageHandler { |
| 83 | public: |
| 84 | virtual ~MediaCodecVideoEncoder(); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 85 | MediaCodecVideoEncoder(JNIEnv* jni, |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 86 | VideoCodecType codecType, |
| 87 | jobject egl_context); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 88 | |
| 89 | // webrtc::VideoEncoder implementation. Everything trampolines to |
| 90 | // |codec_thread_| for execution. |
| 91 | int32_t InitEncode(const webrtc::VideoCodec* codec_settings, |
| 92 | int32_t /* number_of_cores */, |
| 93 | size_t /* max_payload_size */) override; |
pbos | 22993e1 | 2015-10-19 02:39:06 -0700 | [diff] [blame] | 94 | int32_t Encode(const webrtc::VideoFrame& input_image, |
| 95 | const webrtc::CodecSpecificInfo* /* codec_specific_info */, |
| 96 | const std::vector<webrtc::FrameType>* frame_types) override; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 97 | int32_t RegisterEncodeCompleteCallback( |
| 98 | webrtc::EncodedImageCallback* callback) override; |
| 99 | int32_t Release() override; |
| 100 | int32_t SetChannelParameters(uint32_t /* packet_loss */, |
| 101 | int64_t /* rtt */) override; |
| 102 | int32_t SetRates(uint32_t new_bit_rate, uint32_t frame_rate) override; |
| 103 | |
| 104 | // rtc::MessageHandler implementation. |
| 105 | void OnMessage(rtc::Message* msg) override; |
| 106 | |
jackychen | 61b4d51 | 2015-04-21 15:30:11 -0700 | [diff] [blame] | 107 | void OnDroppedFrame() override; |
| 108 | |
jackychen | 6e2ce6e | 2015-07-13 16:26:33 -0700 | [diff] [blame] | 109 | int GetTargetFramerate() override; |
| 110 | |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 111 | bool SupportsNativeHandle() const override { return true; } |
| 112 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 113 | private: |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 114 | // ResetCodecOnCodecThread() calls ReleaseOnCodecThread() and |
| 115 | // InitEncodeOnCodecThread() in an attempt to restore the codec to an |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 116 | // operable state. Necessary after all manner of OMX-layer errors. |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 117 | bool ResetCodecOnCodecThread(); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 118 | |
| 119 | // Implementation of webrtc::VideoEncoder methods above, all running on the |
| 120 | // codec thread exclusively. |
| 121 | // |
| 122 | // If width==0 then this is assumed to be a re-initialization and the |
| 123 | // previously-current values are reused instead of the passed parameters |
| 124 | // (makes it easier to reason about thread-safety). |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 125 | int32_t InitEncodeOnCodecThread(int width, int height, int kbps, int fps, |
| 126 | bool use_surface); |
| 127 | // Reconfigure to match |frame| in width, height. Also reconfigures the |
| 128 | // encoder if |frame| is a texture/byte buffer and the encoder is initialized |
| 129 | // for byte buffer/texture. Returns false if reconfiguring fails. |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 130 | bool MaybeReconfigureEncoderOnCodecThread(const webrtc::VideoFrame& frame); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 131 | int32_t EncodeOnCodecThread( |
Miguel Casas-Sanchez | 4765070 | 2015-05-29 17:21:40 -0700 | [diff] [blame] | 132 | const webrtc::VideoFrame& input_image, |
pbos | 22993e1 | 2015-10-19 02:39:06 -0700 | [diff] [blame] | 133 | const std::vector<webrtc::FrameType>* frame_types); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 134 | bool EncodeByteBufferOnCodecThread(JNIEnv* jni, |
| 135 | bool key_frame, const webrtc::VideoFrame& frame, int input_buffer_index); |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 136 | bool EncodeTextureOnCodecThread(JNIEnv* jni, |
| 137 | bool key_frame, const webrtc::VideoFrame& frame); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 138 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 139 | int32_t RegisterEncodeCompleteCallbackOnCodecThread( |
| 140 | webrtc::EncodedImageCallback* callback); |
| 141 | int32_t ReleaseOnCodecThread(); |
| 142 | int32_t SetRatesOnCodecThread(uint32_t new_bit_rate, uint32_t frame_rate); |
| 143 | |
| 144 | // Helper accessors for MediaCodecVideoEncoder$OutputBufferInfo members. |
| 145 | int GetOutputBufferInfoIndex(JNIEnv* jni, jobject j_output_buffer_info); |
| 146 | jobject GetOutputBufferInfoBuffer(JNIEnv* jni, jobject j_output_buffer_info); |
| 147 | bool GetOutputBufferInfoIsKeyFrame(JNIEnv* jni, jobject j_output_buffer_info); |
| 148 | jlong GetOutputBufferInfoPresentationTimestampUs( |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 149 | JNIEnv* jni, jobject j_output_buffer_info); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 150 | |
| 151 | // Deliver any outputs pending in the MediaCodec to our |callback_| and return |
| 152 | // true on success. |
| 153 | bool DeliverPendingOutputs(JNIEnv* jni); |
| 154 | |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 155 | // Search for H.264 start codes. |
| 156 | int32_t NextNaluPosition(uint8_t *buffer, size_t buffer_size); |
| 157 | |
| 158 | // Type of video codec. |
| 159 | VideoCodecType codecType_; |
| 160 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 161 | // Valid all the time since RegisterEncodeCompleteCallback() Invoke()s to |
| 162 | // |codec_thread_| synchronously. |
| 163 | webrtc::EncodedImageCallback* callback_; |
| 164 | |
| 165 | // State that is constant for the lifetime of this object once the ctor |
| 166 | // returns. |
| 167 | scoped_ptr<Thread> codec_thread_; // Thread on which to operate MediaCodec. |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 168 | rtc::ThreadChecker codec_thread_checker_; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 169 | ScopedGlobalRef<jclass> j_media_codec_video_encoder_class_; |
| 170 | ScopedGlobalRef<jobject> j_media_codec_video_encoder_; |
| 171 | jmethodID j_init_encode_method_; |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 172 | jmethodID j_get_input_buffers_method_; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 173 | jmethodID j_dequeue_input_buffer_method_; |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 174 | jmethodID j_encode_buffer_method_; |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 175 | jmethodID j_encode_texture_method_; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 176 | jmethodID j_release_method_; |
| 177 | jmethodID j_set_rates_method_; |
| 178 | jmethodID j_dequeue_output_buffer_method_; |
| 179 | jmethodID j_release_output_buffer_method_; |
| 180 | jfieldID j_color_format_field_; |
| 181 | jfieldID j_info_index_field_; |
| 182 | jfieldID j_info_buffer_field_; |
| 183 | jfieldID j_info_is_key_frame_field_; |
| 184 | jfieldID j_info_presentation_timestamp_us_field_; |
| 185 | |
| 186 | // State that is valid only between InitEncode() and the next Release(). |
| 187 | // Touched only on codec_thread_ so no explicit synchronization necessary. |
| 188 | int width_; // Frame width in pixels. |
| 189 | int height_; // Frame height in pixels. |
| 190 | bool inited_; |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 191 | bool use_surface_; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 192 | uint16_t picture_id_; |
| 193 | enum libyuv::FourCC encoder_fourcc_; // Encoder color space format. |
| 194 | int last_set_bitrate_kbps_; // Last-requested bitrate in kbps. |
| 195 | int last_set_fps_; // Last-requested frame rate. |
| 196 | int64_t current_timestamp_us_; // Current frame timestamps in us. |
| 197 | int frames_received_; // Number of frames received by encoder. |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 198 | int frames_encoded_; // Number of frames encoded by encoder. |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 199 | int frames_dropped_; // Number of frames dropped by encoder. |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 200 | int frames_in_queue_; // Number of frames in encoder queue. |
| 201 | int64_t start_time_ms_; // Start time for statistics. |
| 202 | int current_frames_; // Number of frames in the current statistics interval. |
| 203 | int current_bytes_; // Encoded bytes in the current statistics interval. |
| 204 | int current_encoding_time_ms_; // Overall encoding time in the current second |
| 205 | int64_t last_input_timestamp_ms_; // Timestamp of last received yuv frame. |
| 206 | int64_t last_output_timestamp_ms_; // Timestamp of last encoded frame. |
| 207 | std::vector<int32_t> timestamps_; // Video frames timestamp queue. |
| 208 | std::vector<int64_t> render_times_ms_; // Video frames render time queue. |
| 209 | std::vector<int64_t> frame_rtc_times_ms_; // Time when video frame is sent to |
| 210 | // encoder input. |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 211 | int32_t output_timestamp_; // Last output frame timestamp from timestamps_ Q. |
| 212 | int64_t output_render_time_ms_; // Last output frame render time from |
| 213 | // render_times_ms_ queue. |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 214 | // Frame size in bytes fed to MediaCodec. |
| 215 | int yuv_size_; |
| 216 | // True only when between a callback_->Encoded() call return a positive value |
| 217 | // and the next Encode() call being ignored. |
| 218 | bool drop_next_input_frame_; |
| 219 | // Global references; must be deleted in Release(). |
| 220 | std::vector<jobject> input_buffers_; |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 221 | webrtc::QualityScaler quality_scaler_; |
jackychen | 61b4d51 | 2015-04-21 15:30:11 -0700 | [diff] [blame] | 222 | // Dynamic resolution change, off by default. |
| 223 | bool scale_; |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 224 | |
| 225 | // H264 bitstream parser, used to extract QP from encoded bitstreams. |
| 226 | webrtc::H264BitstreamParser h264_bitstream_parser_; |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 227 | |
| 228 | // VP9 variables to populate codec specific structure. |
| 229 | webrtc::GofInfoVP9 gof_; // Contains each frame's temporal information for |
| 230 | // non-flexible VP9 mode. |
| 231 | uint8_t tl0_pic_idx_; |
| 232 | size_t gof_idx_; |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 233 | |
| 234 | // EGL context - owned by factory, should not be allocated/destroyed |
| 235 | // by MediaCodecVideoEncoder. |
| 236 | jobject egl_context_; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 237 | }; |
| 238 | |
| 239 | MediaCodecVideoEncoder::~MediaCodecVideoEncoder() { |
| 240 | // Call Release() to ensure no more callbacks to us after we are deleted. |
| 241 | Release(); |
| 242 | } |
| 243 | |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 244 | MediaCodecVideoEncoder::MediaCodecVideoEncoder( |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 245 | JNIEnv* jni, VideoCodecType codecType, jobject egl_context) : |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 246 | codecType_(codecType), |
| 247 | callback_(NULL), |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 248 | inited_(false), |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 249 | use_surface_(false), |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 250 | picture_id_(0), |
| 251 | codec_thread_(new Thread()), |
| 252 | j_media_codec_video_encoder_class_( |
| 253 | jni, |
| 254 | FindClass(jni, "org/webrtc/MediaCodecVideoEncoder")), |
| 255 | j_media_codec_video_encoder_( |
| 256 | jni, |
| 257 | jni->NewObject(*j_media_codec_video_encoder_class_, |
| 258 | GetMethodID(jni, |
| 259 | *j_media_codec_video_encoder_class_, |
| 260 | "<init>", |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 261 | "()V"))), |
| 262 | egl_context_(egl_context) { |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 263 | ScopedLocalRefFrame local_ref_frame(jni); |
| 264 | // It would be nice to avoid spinning up a new thread per MediaCodec, and |
| 265 | // instead re-use e.g. the PeerConnectionFactory's |worker_thread_|, but bug |
| 266 | // 2732 means that deadlocks abound. This class synchronously trampolines |
| 267 | // to |codec_thread_|, so if anything else can be coming to _us_ from |
| 268 | // |codec_thread_|, or from any thread holding the |_sendCritSect| described |
| 269 | // in the bug, we have a problem. For now work around that with a dedicated |
| 270 | // thread. |
| 271 | codec_thread_->SetName("MediaCodecVideoEncoder", NULL); |
henrikg | 91d6ede | 2015-09-17 00:24:34 -0700 | [diff] [blame] | 272 | RTC_CHECK(codec_thread_->Start()) << "Failed to start MediaCodecVideoEncoder"; |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 273 | codec_thread_checker_.DetachFromThread(); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 274 | jclass j_output_buffer_info_class = |
| 275 | FindClass(jni, "org/webrtc/MediaCodecVideoEncoder$OutputBufferInfo"); |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 276 | j_init_encode_method_ = GetMethodID( |
| 277 | jni, |
| 278 | *j_media_codec_video_encoder_class_, |
| 279 | "initEncode", |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 280 | "(Lorg/webrtc/MediaCodecVideoEncoder$VideoCodecType;" |
| 281 | "IIIILjavax/microedition/khronos/egl/EGLContext;)Z"); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 282 | j_get_input_buffers_method_ = GetMethodID( |
| 283 | jni, |
| 284 | *j_media_codec_video_encoder_class_, |
| 285 | "getInputBuffers", |
| 286 | "()[Ljava/nio/ByteBuffer;"); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 287 | j_dequeue_input_buffer_method_ = GetMethodID( |
| 288 | jni, *j_media_codec_video_encoder_class_, "dequeueInputBuffer", "()I"); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 289 | j_encode_buffer_method_ = GetMethodID( |
| 290 | jni, *j_media_codec_video_encoder_class_, "encodeBuffer", "(ZIIJ)Z"); |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 291 | j_encode_texture_method_ = GetMethodID( |
| 292 | jni, *j_media_codec_video_encoder_class_, "encodeTexture", |
| 293 | "(ZI[FJ)Z"); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 294 | j_release_method_ = |
| 295 | GetMethodID(jni, *j_media_codec_video_encoder_class_, "release", "()V"); |
| 296 | j_set_rates_method_ = GetMethodID( |
| 297 | jni, *j_media_codec_video_encoder_class_, "setRates", "(II)Z"); |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 298 | j_dequeue_output_buffer_method_ = GetMethodID( |
| 299 | jni, |
| 300 | *j_media_codec_video_encoder_class_, |
| 301 | "dequeueOutputBuffer", |
| 302 | "()Lorg/webrtc/MediaCodecVideoEncoder$OutputBufferInfo;"); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 303 | j_release_output_buffer_method_ = GetMethodID( |
| 304 | jni, *j_media_codec_video_encoder_class_, "releaseOutputBuffer", "(I)Z"); |
| 305 | |
| 306 | j_color_format_field_ = |
| 307 | GetFieldID(jni, *j_media_codec_video_encoder_class_, "colorFormat", "I"); |
| 308 | j_info_index_field_ = |
| 309 | GetFieldID(jni, j_output_buffer_info_class, "index", "I"); |
| 310 | j_info_buffer_field_ = GetFieldID( |
| 311 | jni, j_output_buffer_info_class, "buffer", "Ljava/nio/ByteBuffer;"); |
| 312 | j_info_is_key_frame_field_ = |
| 313 | GetFieldID(jni, j_output_buffer_info_class, "isKeyFrame", "Z"); |
| 314 | j_info_presentation_timestamp_us_field_ = GetFieldID( |
| 315 | jni, j_output_buffer_info_class, "presentationTimestampUs", "J"); |
| 316 | CHECK_EXCEPTION(jni) << "MediaCodecVideoEncoder ctor failed"; |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 317 | srand(time(NULL)); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 318 | AllowBlockingCalls(); |
| 319 | } |
| 320 | |
| 321 | int32_t MediaCodecVideoEncoder::InitEncode( |
| 322 | const webrtc::VideoCodec* codec_settings, |
| 323 | int32_t /* number_of_cores */, |
| 324 | size_t /* max_payload_size */) { |
jackychen | 61b4d51 | 2015-04-21 15:30:11 -0700 | [diff] [blame] | 325 | const int kMinWidth = 320; |
| 326 | const int kMinHeight = 180; |
jackychen | 98d8cf5 | 2015-05-21 11:12:02 -0700 | [diff] [blame] | 327 | const int kLowQpThresholdDenominator = 3; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 328 | if (codec_settings == NULL) { |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 329 | ALOGE << "NULL VideoCodec instance"; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 330 | return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; |
| 331 | } |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 332 | // Factory should guard against other codecs being used with us. |
henrikg | 91d6ede | 2015-09-17 00:24:34 -0700 | [diff] [blame] | 333 | RTC_CHECK(codec_settings->codecType == codecType_) |
| 334 | << "Unsupported codec " << codec_settings->codecType << " for " |
| 335 | << codecType_; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 336 | |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 337 | ALOGD << "InitEncode request"; |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 338 | scale_ = (codecType_ != kVideoCodecVP9) && (webrtc::field_trial::FindFullName( |
| 339 | "WebRTC-MediaCodecVideoEncoder-AutomaticResize") == "Enabled"); |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 340 | ALOGD << "Encoder automatic resize " << (scale_ ? "enabled" : "disabled"); |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 341 | if (scale_) { |
| 342 | if (codecType_ == kVideoCodecVP8) { |
| 343 | // QP is obtained from VP8-bitstream for HW, so the QP corresponds to the |
| 344 | // (internal) range: [0, 127]. And we cannot change QP_max in HW, so it is |
| 345 | // always = 127. Note that in SW, QP is that of the user-level range [0, |
| 346 | // 63]. |
| 347 | const int kMaxQp = 127; |
Peter Boström | 1741770 | 2015-09-25 17:03:26 +0200 | [diff] [blame] | 348 | // TODO(pbos): Investigate whether high-QP thresholds make sense for VP8. |
| 349 | // This effectively disables high QP as VP8 QP can't go above this |
| 350 | // threshold. |
| 351 | const int kDisabledBadQpThreshold = kMaxQp + 1; |
| 352 | quality_scaler_.Init(kMaxQp / kLowQpThresholdDenominator, |
| 353 | kDisabledBadQpThreshold, true); |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 354 | } else if (codecType_ == kVideoCodecH264) { |
| 355 | // H264 QP is in the range [0, 51]. |
| 356 | const int kMaxQp = 51; |
Peter Boström | 1741770 | 2015-09-25 17:03:26 +0200 | [diff] [blame] | 357 | const int kBadQpThreshold = 40; |
| 358 | quality_scaler_.Init(kMaxQp / kLowQpThresholdDenominator, kBadQpThreshold, |
| 359 | false); |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 360 | } else { |
| 361 | // When adding codec support to additional hardware codecs, also configure |
| 362 | // their QP thresholds for scaling. |
| 363 | RTC_NOTREACHED() << "Unsupported codec without configured QP thresholds."; |
| 364 | } |
| 365 | quality_scaler_.SetMinResolution(kMinWidth, kMinHeight); |
| 366 | quality_scaler_.ReportFramerate(codec_settings->maxFramerate); |
jackychen | 61b4d51 | 2015-04-21 15:30:11 -0700 | [diff] [blame] | 367 | } |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 368 | return codec_thread_->Invoke<int32_t>( |
| 369 | Bind(&MediaCodecVideoEncoder::InitEncodeOnCodecThread, |
| 370 | this, |
| 371 | codec_settings->width, |
| 372 | codec_settings->height, |
| 373 | codec_settings->startBitrate, |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 374 | codec_settings->maxFramerate, |
| 375 | false /* use_surface */)); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 376 | } |
| 377 | |
| 378 | int32_t MediaCodecVideoEncoder::Encode( |
Miguel Casas-Sanchez | 4765070 | 2015-05-29 17:21:40 -0700 | [diff] [blame] | 379 | const webrtc::VideoFrame& frame, |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 380 | const webrtc::CodecSpecificInfo* /* codec_specific_info */, |
pbos | 22993e1 | 2015-10-19 02:39:06 -0700 | [diff] [blame] | 381 | const std::vector<webrtc::FrameType>* frame_types) { |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 382 | return codec_thread_->Invoke<int32_t>(Bind( |
| 383 | &MediaCodecVideoEncoder::EncodeOnCodecThread, this, frame, frame_types)); |
| 384 | } |
| 385 | |
| 386 | int32_t MediaCodecVideoEncoder::RegisterEncodeCompleteCallback( |
| 387 | webrtc::EncodedImageCallback* callback) { |
| 388 | return codec_thread_->Invoke<int32_t>( |
| 389 | Bind(&MediaCodecVideoEncoder::RegisterEncodeCompleteCallbackOnCodecThread, |
| 390 | this, |
| 391 | callback)); |
| 392 | } |
| 393 | |
| 394 | int32_t MediaCodecVideoEncoder::Release() { |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 395 | ALOGD << "EncoderRelease request"; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 396 | return codec_thread_->Invoke<int32_t>( |
| 397 | Bind(&MediaCodecVideoEncoder::ReleaseOnCodecThread, this)); |
| 398 | } |
| 399 | |
| 400 | int32_t MediaCodecVideoEncoder::SetChannelParameters(uint32_t /* packet_loss */, |
| 401 | int64_t /* rtt */) { |
| 402 | return WEBRTC_VIDEO_CODEC_OK; |
| 403 | } |
| 404 | |
| 405 | int32_t MediaCodecVideoEncoder::SetRates(uint32_t new_bit_rate, |
| 406 | uint32_t frame_rate) { |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 407 | if (scale_) |
| 408 | quality_scaler_.ReportFramerate(frame_rate); |
| 409 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 410 | return codec_thread_->Invoke<int32_t>( |
| 411 | Bind(&MediaCodecVideoEncoder::SetRatesOnCodecThread, |
| 412 | this, |
| 413 | new_bit_rate, |
| 414 | frame_rate)); |
| 415 | } |
| 416 | |
| 417 | void MediaCodecVideoEncoder::OnMessage(rtc::Message* msg) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 418 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 419 | JNIEnv* jni = AttachCurrentThreadIfNeeded(); |
| 420 | ScopedLocalRefFrame local_ref_frame(jni); |
| 421 | |
| 422 | // We only ever send one message to |this| directly (not through a Bind()'d |
| 423 | // functor), so expect no ID/data. |
henrikg | 91d6ede | 2015-09-17 00:24:34 -0700 | [diff] [blame] | 424 | RTC_CHECK(!msg->message_id) << "Unexpected message!"; |
| 425 | RTC_CHECK(!msg->pdata) << "Unexpected message!"; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 426 | if (!inited_) { |
| 427 | return; |
| 428 | } |
| 429 | |
| 430 | // It would be nice to recover from a failure here if one happened, but it's |
| 431 | // unclear how to signal such a failure to the app, so instead we stay silent |
| 432 | // about it and let the next app-called API method reveal the borkedness. |
| 433 | DeliverPendingOutputs(jni); |
| 434 | codec_thread_->PostDelayed(kMediaCodecPollMs, this); |
| 435 | } |
| 436 | |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 437 | bool MediaCodecVideoEncoder::ResetCodecOnCodecThread() { |
| 438 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
| 439 | ALOGE << "ResetOnCodecThread"; |
| 440 | if (ReleaseOnCodecThread() != WEBRTC_VIDEO_CODEC_OK || |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 441 | InitEncodeOnCodecThread(width_, height_, 0, 0, false) != |
| 442 | WEBRTC_VIDEO_CODEC_OK) { |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 443 | // TODO(fischman): wouldn't it be nice if there was a way to gracefully |
| 444 | // degrade to a SW encoder at this point? There isn't one AFAICT :( |
| 445 | // https://code.google.com/p/webrtc/issues/detail?id=2920 |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 446 | return false; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 447 | } |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 448 | return true; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 449 | } |
| 450 | |
| 451 | int32_t MediaCodecVideoEncoder::InitEncodeOnCodecThread( |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 452 | int width, int height, int kbps, int fps, bool use_surface) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 453 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 454 | RTC_CHECK(!use_surface || egl_context_ != nullptr) << "EGL context not set."; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 455 | JNIEnv* jni = AttachCurrentThreadIfNeeded(); |
| 456 | ScopedLocalRefFrame local_ref_frame(jni); |
| 457 | |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 458 | ALOGD << "InitEncodeOnCodecThread Type: " << (int)codecType_ << ", " << |
| 459 | width << " x " << height << ". Bitrate: " << kbps << |
| 460 | " kbps. Fps: " << fps; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 461 | if (kbps == 0) { |
| 462 | kbps = last_set_bitrate_kbps_; |
| 463 | } |
| 464 | if (fps == 0) { |
| 465 | fps = last_set_fps_; |
| 466 | } |
| 467 | |
| 468 | width_ = width; |
| 469 | height_ = height; |
| 470 | last_set_bitrate_kbps_ = kbps; |
| 471 | last_set_fps_ = fps; |
| 472 | yuv_size_ = width_ * height_ * 3 / 2; |
| 473 | frames_received_ = 0; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 474 | frames_encoded_ = 0; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 475 | frames_dropped_ = 0; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 476 | frames_in_queue_ = 0; |
| 477 | current_timestamp_us_ = 0; |
| 478 | start_time_ms_ = GetCurrentTimeMs(); |
| 479 | current_frames_ = 0; |
| 480 | current_bytes_ = 0; |
| 481 | current_encoding_time_ms_ = 0; |
| 482 | last_input_timestamp_ms_ = -1; |
| 483 | last_output_timestamp_ms_ = -1; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 484 | output_timestamp_ = 0; |
| 485 | output_render_time_ms_ = 0; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 486 | timestamps_.clear(); |
| 487 | render_times_ms_.clear(); |
| 488 | frame_rtc_times_ms_.clear(); |
| 489 | drop_next_input_frame_ = false; |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 490 | use_surface_ = use_surface; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 491 | picture_id_ = static_cast<uint16_t>(rand()) & 0x7FFF; |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 492 | gof_.SetGofInfoVP9(webrtc::TemporalStructureMode::kTemporalStructureMode1); |
| 493 | tl0_pic_idx_ = static_cast<uint8_t>(rand()); |
| 494 | gof_idx_ = 0; |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 495 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 496 | // We enforce no extra stride/padding in the format creation step. |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 497 | jobject j_video_codec_enum = JavaEnumFromIndex( |
| 498 | jni, "MediaCodecVideoEncoder$VideoCodecType", codecType_); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 499 | const bool encode_status = jni->CallBooleanMethod( |
| 500 | *j_media_codec_video_encoder_, j_init_encode_method_, |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 501 | j_video_codec_enum, width, height, kbps, fps, |
| 502 | (use_surface ? egl_context_ : nullptr)); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 503 | if (!encode_status) { |
| 504 | ALOGE << "Failed to configure encoder."; |
| 505 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 506 | } |
| 507 | CHECK_EXCEPTION(jni); |
| 508 | |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 509 | if (use_surface) { |
| 510 | scale_ = false; // TODO(perkj): Implement scaling when using textures. |
| 511 | } else { |
| 512 | jobjectArray input_buffers = reinterpret_cast<jobjectArray>( |
| 513 | jni->CallObjectMethod(*j_media_codec_video_encoder_, |
| 514 | j_get_input_buffers_method_)); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 515 | CHECK_EXCEPTION(jni); |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 516 | if (IsNull(jni, input_buffers)) { |
| 517 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 518 | } |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 519 | |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 520 | switch (GetIntField(jni, *j_media_codec_video_encoder_, |
| 521 | j_color_format_field_)) { |
| 522 | case COLOR_FormatYUV420Planar: |
| 523 | encoder_fourcc_ = libyuv::FOURCC_YU12; |
| 524 | break; |
| 525 | case COLOR_FormatYUV420SemiPlanar: |
| 526 | case COLOR_QCOM_FormatYUV420SemiPlanar: |
| 527 | case COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m: |
| 528 | encoder_fourcc_ = libyuv::FOURCC_NV12; |
| 529 | break; |
| 530 | default: |
| 531 | LOG(LS_ERROR) << "Wrong color format."; |
| 532 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 533 | } |
| 534 | size_t num_input_buffers = jni->GetArrayLength(input_buffers); |
| 535 | RTC_CHECK(input_buffers_.empty()) |
| 536 | << "Unexpected double InitEncode without Release"; |
| 537 | input_buffers_.resize(num_input_buffers); |
| 538 | for (size_t i = 0; i < num_input_buffers; ++i) { |
| 539 | input_buffers_[i] = |
| 540 | jni->NewGlobalRef(jni->GetObjectArrayElement(input_buffers, i)); |
| 541 | int64_t yuv_buffer_capacity = |
| 542 | jni->GetDirectBufferCapacity(input_buffers_[i]); |
| 543 | CHECK_EXCEPTION(jni); |
| 544 | RTC_CHECK(yuv_buffer_capacity >= yuv_size_) << "Insufficient capacity"; |
| 545 | } |
| 546 | } |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 547 | |
| 548 | inited_ = true; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 549 | codec_thread_->PostDelayed(kMediaCodecPollMs, this); |
| 550 | return WEBRTC_VIDEO_CODEC_OK; |
| 551 | } |
| 552 | |
| 553 | int32_t MediaCodecVideoEncoder::EncodeOnCodecThread( |
Miguel Casas-Sanchez | 4765070 | 2015-05-29 17:21:40 -0700 | [diff] [blame] | 554 | const webrtc::VideoFrame& frame, |
pbos | 22993e1 | 2015-10-19 02:39:06 -0700 | [diff] [blame] | 555 | const std::vector<webrtc::FrameType>* frame_types) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 556 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 557 | JNIEnv* jni = AttachCurrentThreadIfNeeded(); |
| 558 | ScopedLocalRefFrame local_ref_frame(jni); |
| 559 | |
| 560 | if (!inited_) { |
| 561 | return WEBRTC_VIDEO_CODEC_UNINITIALIZED; |
| 562 | } |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 563 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 564 | frames_received_++; |
| 565 | if (!DeliverPendingOutputs(jni)) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 566 | if (!ResetCodecOnCodecThread()) |
| 567 | return WEBRTC_VIDEO_CODEC_ERROR; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 568 | } |
| 569 | |
| 570 | if (drop_next_input_frame_) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 571 | ALOGW << "Encoder drop frame - failed callback."; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 572 | drop_next_input_frame_ = false; |
| 573 | return WEBRTC_VIDEO_CODEC_OK; |
| 574 | } |
| 575 | |
henrikg | 91d6ede | 2015-09-17 00:24:34 -0700 | [diff] [blame] | 576 | RTC_CHECK(frame_types->size() == 1) << "Unexpected stream count"; |
jackychen | 6e2ce6e | 2015-07-13 16:26:33 -0700 | [diff] [blame] | 577 | // Check framerate before spatial resolution change. |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 578 | if (scale_) |
| 579 | quality_scaler_.OnEncodeFrame(frame); |
| 580 | |
| 581 | const VideoFrame& input_frame = |
| 582 | scale_ ? quality_scaler_.GetScaledFrame(frame) : frame; |
jackychen | 61b4d51 | 2015-04-21 15:30:11 -0700 | [diff] [blame] | 583 | |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 584 | if (!MaybeReconfigureEncoderOnCodecThread(input_frame)) { |
| 585 | ALOGE << "Failed to reconfigure encoder."; |
| 586 | return WEBRTC_VIDEO_CODEC_ERROR; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 587 | } |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 588 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 589 | // Check if we accumulated too many frames in encoder input buffers |
| 590 | // or the encoder latency exceeds 70 ms and drop frame if so. |
| 591 | if (frames_in_queue_ > 0 && last_input_timestamp_ms_ >= 0) { |
| 592 | int encoder_latency_ms = last_input_timestamp_ms_ - |
| 593 | last_output_timestamp_ms_; |
| 594 | if (frames_in_queue_ > 2 || encoder_latency_ms > 70) { |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 595 | ALOGD << "Drop frame - encoder is behind by " << encoder_latency_ms << |
| 596 | " ms. Q size: " << frames_in_queue_; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 597 | frames_dropped_++; |
jackychen | 61b4d51 | 2015-04-21 15:30:11 -0700 | [diff] [blame] | 598 | // Report dropped frame to quality_scaler_. |
| 599 | OnDroppedFrame(); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 600 | return WEBRTC_VIDEO_CODEC_OK; |
| 601 | } |
| 602 | } |
| 603 | |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 604 | const bool key_frame = frame_types->front() != webrtc::kVideoFrameDelta; |
| 605 | bool encode_status = true; |
| 606 | if (!input_frame.native_handle()) { |
| 607 | int j_input_buffer_index = jni->CallIntMethod(*j_media_codec_video_encoder_, |
| 608 | j_dequeue_input_buffer_method_); |
| 609 | CHECK_EXCEPTION(jni); |
| 610 | if (j_input_buffer_index == -1) { |
| 611 | // Video codec falls behind - no input buffer available. |
| 612 | ALOGW << "Encoder drop frame - no input buffers available"; |
| 613 | frames_dropped_++; |
| 614 | // Report dropped frame to quality_scaler_. |
| 615 | OnDroppedFrame(); |
| 616 | return WEBRTC_VIDEO_CODEC_OK; // TODO(fischman): see webrtc bug 2887. |
| 617 | } |
| 618 | if (j_input_buffer_index == -2) { |
| 619 | ResetCodecOnCodecThread(); |
| 620 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 621 | } |
| 622 | encode_status = EncodeByteBufferOnCodecThread(jni, key_frame, input_frame, |
| 623 | j_input_buffer_index); |
| 624 | } else { |
| 625 | encode_status = EncodeTextureOnCodecThread(jni, key_frame, input_frame); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 626 | } |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 627 | |
| 628 | if (!encode_status) { |
| 629 | ALOGE << "Failed encode frame with timestamp: " << input_frame.timestamp(); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 630 | ResetCodecOnCodecThread(); |
perkj | 12f6802 | 2015-10-16 13:31:45 +0200 | [diff] [blame] | 631 | return WEBRTC_VIDEO_CODEC_ERROR; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 632 | } |
| 633 | |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 634 | last_input_timestamp_ms_ = |
| 635 | current_timestamp_us_ / rtc::kNumMicrosecsPerMillisec; |
perkj | 12f6802 | 2015-10-16 13:31:45 +0200 | [diff] [blame] | 636 | frames_in_queue_++; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 637 | |
perkj | 12f6802 | 2015-10-16 13:31:45 +0200 | [diff] [blame] | 638 | // Save input image timestamps for later output |
| 639 | timestamps_.push_back(input_frame.timestamp()); |
| 640 | render_times_ms_.push_back(input_frame.render_time_ms()); |
| 641 | frame_rtc_times_ms_.push_back(GetCurrentTimeMs()); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 642 | current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_; |
| 643 | |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 644 | if (!DeliverPendingOutputs(jni)) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 645 | ALOGE << "Failed deliver pending outputs."; |
| 646 | ResetCodecOnCodecThread(); |
| 647 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 648 | } |
| 649 | return WEBRTC_VIDEO_CODEC_OK; |
| 650 | } |
| 651 | |
| 652 | bool MediaCodecVideoEncoder::MaybeReconfigureEncoderOnCodecThread( |
| 653 | const webrtc::VideoFrame& frame) { |
| 654 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
| 655 | |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 656 | const bool is_texture_frame = frame.native_handle() != nullptr; |
| 657 | const bool reconfigure_due_to_format = is_texture_frame != use_surface_; |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 658 | const bool reconfigure_due_to_size = |
| 659 | frame.width() != width_ || frame.height() != height_; |
| 660 | |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 661 | if (reconfigure_due_to_format) { |
| 662 | ALOGD << "Reconfigure encoder due to format change. " |
| 663 | << (use_surface_ ? |
| 664 | "Reconfiguring to encode from byte buffer." : |
| 665 | "Reconfiguring to encode from texture."); |
| 666 | } |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 667 | if (reconfigure_due_to_size) { |
| 668 | ALOGD << "Reconfigure encoder due to frame resolution change from " |
| 669 | << width_ << " x " << height_ << " to " << frame.width() << " x " |
| 670 | << frame.height(); |
| 671 | width_ = frame.width(); |
| 672 | height_ = frame.height(); |
| 673 | } |
| 674 | |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 675 | if (!reconfigure_due_to_format && !reconfigure_due_to_size) |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 676 | return true; |
| 677 | |
| 678 | ReleaseOnCodecThread(); |
| 679 | |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 680 | return InitEncodeOnCodecThread(width_, height_, 0, 0 , is_texture_frame) == |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 681 | WEBRTC_VIDEO_CODEC_OK; |
| 682 | } |
| 683 | |
| 684 | bool MediaCodecVideoEncoder::EncodeByteBufferOnCodecThread(JNIEnv* jni, |
| 685 | bool key_frame, const webrtc::VideoFrame& frame, int input_buffer_index) { |
| 686 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 687 | RTC_CHECK(!use_surface_); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 688 | |
| 689 | ALOGV("Encoder frame in # %d. TS: %lld. Q: %d", |
| 690 | frames_received_ - 1, current_timestamp_us_ / 1000, frames_in_queue_); |
| 691 | |
| 692 | jobject j_input_buffer = input_buffers_[input_buffer_index]; |
| 693 | uint8_t* yuv_buffer = |
| 694 | reinterpret_cast<uint8_t*>(jni->GetDirectBufferAddress(j_input_buffer)); |
| 695 | CHECK_EXCEPTION(jni); |
| 696 | RTC_CHECK(yuv_buffer) << "Indirect buffer??"; |
| 697 | RTC_CHECK(!libyuv::ConvertFromI420( |
| 698 | frame.buffer(webrtc::kYPlane), frame.stride(webrtc::kYPlane), |
| 699 | frame.buffer(webrtc::kUPlane), frame.stride(webrtc::kUPlane), |
| 700 | frame.buffer(webrtc::kVPlane), frame.stride(webrtc::kVPlane), |
| 701 | yuv_buffer, width_, width_, height_, encoder_fourcc_)) |
| 702 | << "ConvertFromI420 failed"; |
| 703 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 704 | bool encode_status = jni->CallBooleanMethod(*j_media_codec_video_encoder_, |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 705 | j_encode_buffer_method_, |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 706 | key_frame, |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 707 | input_buffer_index, |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 708 | yuv_size_, |
| 709 | current_timestamp_us_); |
| 710 | CHECK_EXCEPTION(jni); |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 711 | return encode_status; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 712 | } |
| 713 | |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 714 | bool MediaCodecVideoEncoder::EncodeTextureOnCodecThread(JNIEnv* jni, |
| 715 | bool key_frame, const webrtc::VideoFrame& frame) { |
| 716 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
| 717 | RTC_CHECK(use_surface_); |
| 718 | NativeHandleImpl* handle = |
| 719 | static_cast<NativeHandleImpl*>(frame.native_handle()); |
| 720 | jfloatArray sampling_matrix = jni->NewFloatArray(16); |
| 721 | jni->SetFloatArrayRegion(sampling_matrix, 0, 16, handle->sampling_matrix); |
| 722 | |
| 723 | bool encode_status = jni->CallBooleanMethod(*j_media_codec_video_encoder_, |
| 724 | j_encode_texture_method_, |
| 725 | key_frame, |
| 726 | handle->oes_texture_id, |
| 727 | sampling_matrix, |
| 728 | current_timestamp_us_); |
| 729 | CHECK_EXCEPTION(jni); |
| 730 | return encode_status; |
| 731 | } |
| 732 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 733 | int32_t MediaCodecVideoEncoder::RegisterEncodeCompleteCallbackOnCodecThread( |
| 734 | webrtc::EncodedImageCallback* callback) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 735 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 736 | JNIEnv* jni = AttachCurrentThreadIfNeeded(); |
| 737 | ScopedLocalRefFrame local_ref_frame(jni); |
| 738 | callback_ = callback; |
| 739 | return WEBRTC_VIDEO_CODEC_OK; |
| 740 | } |
| 741 | |
| 742 | int32_t MediaCodecVideoEncoder::ReleaseOnCodecThread() { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 743 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 744 | if (!inited_) { |
| 745 | return WEBRTC_VIDEO_CODEC_OK; |
| 746 | } |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 747 | JNIEnv* jni = AttachCurrentThreadIfNeeded(); |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 748 | ALOGD << "EncoderReleaseOnCodecThread: Frames received: " << |
| 749 | frames_received_ << ". Encoded: " << frames_encoded_ << |
| 750 | ". Dropped: " << frames_dropped_; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 751 | ScopedLocalRefFrame local_ref_frame(jni); |
| 752 | for (size_t i = 0; i < input_buffers_.size(); ++i) |
| 753 | jni->DeleteGlobalRef(input_buffers_[i]); |
| 754 | input_buffers_.clear(); |
| 755 | jni->CallVoidMethod(*j_media_codec_video_encoder_, j_release_method_); |
| 756 | CHECK_EXCEPTION(jni); |
| 757 | rtc::MessageQueueManager::Clear(this); |
| 758 | inited_ = false; |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 759 | use_surface_ = false; |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 760 | ALOGD << "EncoderReleaseOnCodecThread done."; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 761 | return WEBRTC_VIDEO_CODEC_OK; |
| 762 | } |
| 763 | |
| 764 | int32_t MediaCodecVideoEncoder::SetRatesOnCodecThread(uint32_t new_bit_rate, |
| 765 | uint32_t frame_rate) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 766 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 767 | if (last_set_bitrate_kbps_ == new_bit_rate && |
| 768 | last_set_fps_ == frame_rate) { |
| 769 | return WEBRTC_VIDEO_CODEC_OK; |
| 770 | } |
| 771 | JNIEnv* jni = AttachCurrentThreadIfNeeded(); |
| 772 | ScopedLocalRefFrame local_ref_frame(jni); |
| 773 | if (new_bit_rate > 0) { |
| 774 | last_set_bitrate_kbps_ = new_bit_rate; |
| 775 | } |
| 776 | if (frame_rate > 0) { |
| 777 | last_set_fps_ = frame_rate; |
| 778 | } |
| 779 | bool ret = jni->CallBooleanMethod(*j_media_codec_video_encoder_, |
| 780 | j_set_rates_method_, |
| 781 | last_set_bitrate_kbps_, |
| 782 | last_set_fps_); |
| 783 | CHECK_EXCEPTION(jni); |
| 784 | if (!ret) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 785 | ResetCodecOnCodecThread(); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 786 | return WEBRTC_VIDEO_CODEC_ERROR; |
| 787 | } |
| 788 | return WEBRTC_VIDEO_CODEC_OK; |
| 789 | } |
| 790 | |
| 791 | int MediaCodecVideoEncoder::GetOutputBufferInfoIndex( |
| 792 | JNIEnv* jni, |
| 793 | jobject j_output_buffer_info) { |
| 794 | return GetIntField(jni, j_output_buffer_info, j_info_index_field_); |
| 795 | } |
| 796 | |
| 797 | jobject MediaCodecVideoEncoder::GetOutputBufferInfoBuffer( |
| 798 | JNIEnv* jni, |
| 799 | jobject j_output_buffer_info) { |
| 800 | return GetObjectField(jni, j_output_buffer_info, j_info_buffer_field_); |
| 801 | } |
| 802 | |
| 803 | bool MediaCodecVideoEncoder::GetOutputBufferInfoIsKeyFrame( |
| 804 | JNIEnv* jni, |
| 805 | jobject j_output_buffer_info) { |
| 806 | return GetBooleanField(jni, j_output_buffer_info, j_info_is_key_frame_field_); |
| 807 | } |
| 808 | |
| 809 | jlong MediaCodecVideoEncoder::GetOutputBufferInfoPresentationTimestampUs( |
| 810 | JNIEnv* jni, |
| 811 | jobject j_output_buffer_info) { |
| 812 | return GetLongField( |
| 813 | jni, j_output_buffer_info, j_info_presentation_timestamp_us_field_); |
| 814 | } |
| 815 | |
| 816 | bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 817 | RTC_DCHECK(codec_thread_checker_.CalledOnValidThread()); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 818 | while (true) { |
| 819 | jobject j_output_buffer_info = jni->CallObjectMethod( |
| 820 | *j_media_codec_video_encoder_, j_dequeue_output_buffer_method_); |
| 821 | CHECK_EXCEPTION(jni); |
| 822 | if (IsNull(jni, j_output_buffer_info)) { |
| 823 | break; |
| 824 | } |
| 825 | |
| 826 | int output_buffer_index = |
| 827 | GetOutputBufferInfoIndex(jni, j_output_buffer_info); |
| 828 | if (output_buffer_index == -1) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 829 | ResetCodecOnCodecThread(); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 830 | return false; |
| 831 | } |
| 832 | |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 833 | // Get key and config frame flags. |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 834 | jobject j_output_buffer = |
| 835 | GetOutputBufferInfoBuffer(jni, j_output_buffer_info); |
| 836 | bool key_frame = GetOutputBufferInfoIsKeyFrame(jni, j_output_buffer_info); |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 837 | |
| 838 | // Get frame timestamps from a queue - for non config frames only. |
| 839 | int64_t frame_encoding_time_ms = 0; |
| 840 | last_output_timestamp_ms_ = |
| 841 | GetOutputBufferInfoPresentationTimestampUs(jni, j_output_buffer_info) / |
| 842 | 1000; |
| 843 | if (frames_in_queue_ > 0) { |
| 844 | output_timestamp_ = timestamps_.front(); |
| 845 | timestamps_.erase(timestamps_.begin()); |
| 846 | output_render_time_ms_ = render_times_ms_.front(); |
| 847 | render_times_ms_.erase(render_times_ms_.begin()); |
| 848 | frame_encoding_time_ms = GetCurrentTimeMs() - frame_rtc_times_ms_.front(); |
| 849 | frame_rtc_times_ms_.erase(frame_rtc_times_ms_.begin()); |
| 850 | frames_in_queue_--; |
| 851 | } |
| 852 | |
| 853 | // Extract payload. |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 854 | size_t payload_size = jni->GetDirectBufferCapacity(j_output_buffer); |
Peter Boström | 0c4e06b | 2015-10-07 12:23:21 +0200 | [diff] [blame] | 855 | uint8_t* payload = reinterpret_cast<uint8_t*>( |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 856 | jni->GetDirectBufferAddress(j_output_buffer)); |
| 857 | CHECK_EXCEPTION(jni); |
| 858 | |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 859 | ALOGV("Encoder frame out # %d. Key: %d. Size: %d. TS: %lld." |
| 860 | " Latency: %lld. EncTime: %lld", |
| 861 | frames_encoded_, key_frame, payload_size, |
| 862 | last_output_timestamp_ms_, |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 863 | last_input_timestamp_ms_ - last_output_timestamp_ms_, |
| 864 | frame_encoding_time_ms); |
| 865 | |
| 866 | // Calculate and print encoding statistics - every 3 seconds. |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 867 | frames_encoded_++; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 868 | current_frames_++; |
| 869 | current_bytes_ += payload_size; |
| 870 | current_encoding_time_ms_ += frame_encoding_time_ms; |
| 871 | int statistic_time_ms = GetCurrentTimeMs() - start_time_ms_; |
| 872 | if (statistic_time_ms >= kMediaCodecStatisticsIntervalMs && |
| 873 | current_frames_ > 0) { |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 874 | ALOGD << "Encoded frames: " << frames_encoded_ << ". Bitrate: " << |
| 875 | (current_bytes_ * 8 / statistic_time_ms) << |
| 876 | ", target: " << last_set_bitrate_kbps_ << " kbps, fps: " << |
| 877 | ((current_frames_ * 1000 + statistic_time_ms / 2) / statistic_time_ms) |
| 878 | << ", encTime: " << |
| 879 | (current_encoding_time_ms_ / current_frames_) << " for last " << |
| 880 | statistic_time_ms << " ms."; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 881 | start_time_ms_ = GetCurrentTimeMs(); |
| 882 | current_frames_ = 0; |
| 883 | current_bytes_ = 0; |
| 884 | current_encoding_time_ms_ = 0; |
| 885 | } |
| 886 | |
| 887 | // Callback - return encoded frame. |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 888 | int32_t callback_status = 0; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 889 | if (callback_) { |
| 890 | scoped_ptr<webrtc::EncodedImage> image( |
| 891 | new webrtc::EncodedImage(payload, payload_size, payload_size)); |
| 892 | image->_encodedWidth = width_; |
| 893 | image->_encodedHeight = height_; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 894 | image->_timeStamp = output_timestamp_; |
| 895 | image->capture_time_ms_ = output_render_time_ms_; |
Peter Boström | 49e196a | 2015-10-23 15:58:18 +0200 | [diff] [blame] | 896 | image->_frameType = |
| 897 | (key_frame ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 898 | image->_completeFrame = true; |
asapersson | 075fb4b | 2015-10-29 08:49:14 -0700 | [diff] [blame] | 899 | image->adapt_reason_.quality_resolution_downscales = |
| 900 | scale_ ? quality_scaler_.downscale_shift() : -1; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 901 | |
| 902 | webrtc::CodecSpecificInfo info; |
| 903 | memset(&info, 0, sizeof(info)); |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 904 | info.codecType = codecType_; |
| 905 | if (codecType_ == kVideoCodecVP8) { |
| 906 | info.codecSpecific.VP8.pictureId = picture_id_; |
| 907 | info.codecSpecific.VP8.nonReference = false; |
| 908 | info.codecSpecific.VP8.simulcastIdx = 0; |
| 909 | info.codecSpecific.VP8.temporalIdx = webrtc::kNoTemporalIdx; |
| 910 | info.codecSpecific.VP8.layerSync = false; |
| 911 | info.codecSpecific.VP8.tl0PicIdx = webrtc::kNoTl0PicIdx; |
| 912 | info.codecSpecific.VP8.keyIdx = webrtc::kNoKeyIdx; |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 913 | } else if (codecType_ == kVideoCodecVP9) { |
| 914 | if (key_frame) { |
| 915 | gof_idx_ = 0; |
| 916 | } |
| 917 | info.codecSpecific.VP9.picture_id = picture_id_; |
| 918 | info.codecSpecific.VP9.inter_pic_predicted = key_frame ? false : true; |
| 919 | info.codecSpecific.VP9.flexible_mode = false; |
| 920 | info.codecSpecific.VP9.ss_data_available = key_frame ? true : false; |
| 921 | info.codecSpecific.VP9.tl0_pic_idx = tl0_pic_idx_++; |
| 922 | info.codecSpecific.VP9.temporal_idx = webrtc::kNoTemporalIdx; |
| 923 | info.codecSpecific.VP9.spatial_idx = webrtc::kNoSpatialIdx; |
| 924 | info.codecSpecific.VP9.temporal_up_switch = true; |
| 925 | info.codecSpecific.VP9.inter_layer_predicted = false; |
| 926 | info.codecSpecific.VP9.gof_idx = |
| 927 | static_cast<uint8_t>(gof_idx_++ % gof_.num_frames_in_gof); |
| 928 | info.codecSpecific.VP9.num_spatial_layers = 1; |
| 929 | info.codecSpecific.VP9.spatial_layer_resolution_present = false; |
| 930 | if (info.codecSpecific.VP9.ss_data_available) { |
| 931 | info.codecSpecific.VP9.spatial_layer_resolution_present = true; |
| 932 | info.codecSpecific.VP9.width[0] = width_; |
| 933 | info.codecSpecific.VP9.height[0] = height_; |
| 934 | info.codecSpecific.VP9.gof.CopyGofInfoVP9(gof_); |
| 935 | } |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 936 | } |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 937 | picture_id_ = (picture_id_ + 1) & 0x7FFF; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 938 | |
| 939 | // Generate a header describing a single fragment. |
| 940 | webrtc::RTPFragmentationHeader header; |
| 941 | memset(&header, 0, sizeof(header)); |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 942 | if (codecType_ == kVideoCodecVP8 || codecType_ == kVideoCodecVP9) { |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 943 | header.VerifyAndAllocateFragmentationHeader(1); |
| 944 | header.fragmentationOffset[0] = 0; |
| 945 | header.fragmentationLength[0] = image->_length; |
| 946 | header.fragmentationPlType[0] = 0; |
| 947 | header.fragmentationTimeDiff[0] = 0; |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 948 | if (codecType_ == kVideoCodecVP8 && scale_) { |
asapersson | 86b0160 | 2015-10-20 23:55:26 -0700 | [diff] [blame] | 949 | int qp; |
| 950 | if (webrtc::vp8::GetQp(payload, payload_size, &qp)) |
| 951 | quality_scaler_.ReportQP(qp); |
| 952 | } |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 953 | } else if (codecType_ == kVideoCodecH264) { |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 954 | if (scale_) { |
| 955 | h264_bitstream_parser_.ParseBitstream(payload, payload_size); |
| 956 | int qp; |
| 957 | if (h264_bitstream_parser_.GetLastSliceQp(&qp)) |
| 958 | quality_scaler_.ReportQP(qp); |
| 959 | } |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 960 | // For H.264 search for start codes. |
| 961 | int32_t scPositions[MAX_NALUS_PERFRAME + 1] = {}; |
| 962 | int32_t scPositionsLength = 0; |
| 963 | int32_t scPosition = 0; |
| 964 | while (scPositionsLength < MAX_NALUS_PERFRAME) { |
| 965 | int32_t naluPosition = NextNaluPosition( |
| 966 | payload + scPosition, payload_size - scPosition); |
| 967 | if (naluPosition < 0) { |
| 968 | break; |
| 969 | } |
| 970 | scPosition += naluPosition; |
| 971 | scPositions[scPositionsLength++] = scPosition; |
| 972 | scPosition += H264_SC_LENGTH; |
| 973 | } |
| 974 | if (scPositionsLength == 0) { |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 975 | ALOGE << "Start code is not found!"; |
| 976 | ALOGE << "Data:" << image->_buffer[0] << " " << image->_buffer[1] |
| 977 | << " " << image->_buffer[2] << " " << image->_buffer[3] |
| 978 | << " " << image->_buffer[4] << " " << image->_buffer[5]; |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 979 | ResetCodecOnCodecThread(); |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 980 | return false; |
| 981 | } |
| 982 | scPositions[scPositionsLength] = payload_size; |
| 983 | header.VerifyAndAllocateFragmentationHeader(scPositionsLength); |
| 984 | for (size_t i = 0; i < scPositionsLength; i++) { |
| 985 | header.fragmentationOffset[i] = scPositions[i] + H264_SC_LENGTH; |
| 986 | header.fragmentationLength[i] = |
| 987 | scPositions[i + 1] - header.fragmentationOffset[i]; |
| 988 | header.fragmentationPlType[i] = 0; |
| 989 | header.fragmentationTimeDiff[i] = 0; |
| 990 | } |
| 991 | } |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 992 | |
| 993 | callback_status = callback_->Encoded(*image, &info, &header); |
| 994 | } |
| 995 | |
| 996 | // Return output buffer back to the encoder. |
| 997 | bool success = jni->CallBooleanMethod(*j_media_codec_video_encoder_, |
| 998 | j_release_output_buffer_method_, |
| 999 | output_buffer_index); |
| 1000 | CHECK_EXCEPTION(jni); |
| 1001 | if (!success) { |
perkj | 9576e54 | 2015-11-12 06:43:16 -0800 | [diff] [blame] | 1002 | ResetCodecOnCodecThread(); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1003 | return false; |
| 1004 | } |
| 1005 | |
| 1006 | if (callback_status > 0) { |
| 1007 | drop_next_input_frame_ = true; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1008 | // Theoretically could handle callback_status<0 here, but unclear what |
| 1009 | // that would mean for us. |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1010 | } |
| 1011 | } |
| 1012 | |
| 1013 | return true; |
| 1014 | } |
| 1015 | |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1016 | int32_t MediaCodecVideoEncoder::NextNaluPosition( |
| 1017 | uint8_t *buffer, size_t buffer_size) { |
| 1018 | if (buffer_size < H264_SC_LENGTH) { |
| 1019 | return -1; |
| 1020 | } |
| 1021 | uint8_t *head = buffer; |
| 1022 | // Set end buffer pointer to 4 bytes before actual buffer end so we can |
| 1023 | // access head[1], head[2] and head[3] in a loop without buffer overrun. |
| 1024 | uint8_t *end = buffer + buffer_size - H264_SC_LENGTH; |
| 1025 | |
| 1026 | while (head < end) { |
| 1027 | if (head[0]) { |
| 1028 | head++; |
| 1029 | continue; |
| 1030 | } |
| 1031 | if (head[1]) { // got 00xx |
| 1032 | head += 2; |
| 1033 | continue; |
| 1034 | } |
| 1035 | if (head[2]) { // got 0000xx |
| 1036 | head += 3; |
| 1037 | continue; |
| 1038 | } |
| 1039 | if (head[3] != 0x01) { // got 000000xx |
glaznev@webrtc.org | dc08a23 | 2015-03-06 23:32:20 +0000 | [diff] [blame] | 1040 | head++; // xx != 1, continue searching. |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1041 | continue; |
| 1042 | } |
| 1043 | return (int32_t)(head - buffer); |
| 1044 | } |
| 1045 | return -1; |
| 1046 | } |
| 1047 | |
jackychen | 61b4d51 | 2015-04-21 15:30:11 -0700 | [diff] [blame] | 1048 | void MediaCodecVideoEncoder::OnDroppedFrame() { |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 1049 | if (scale_) |
| 1050 | quality_scaler_.ReportDroppedFrame(); |
jackychen | 61b4d51 | 2015-04-21 15:30:11 -0700 | [diff] [blame] | 1051 | } |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1052 | |
jackychen | 6e2ce6e | 2015-07-13 16:26:33 -0700 | [diff] [blame] | 1053 | int MediaCodecVideoEncoder::GetTargetFramerate() { |
Peter Boström | 2bc68c7 | 2015-09-24 16:22:28 +0200 | [diff] [blame] | 1054 | return scale_ ? quality_scaler_.GetTargetFramerate() : -1; |
jackychen | 6e2ce6e | 2015-07-13 16:26:33 -0700 | [diff] [blame] | 1055 | } |
| 1056 | |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 1057 | MediaCodecVideoEncoderFactory::MediaCodecVideoEncoderFactory() |
| 1058 | : egl_context_ (nullptr) { |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1059 | JNIEnv* jni = AttachCurrentThreadIfNeeded(); |
| 1060 | ScopedLocalRefFrame local_ref_frame(jni); |
| 1061 | jclass j_encoder_class = FindClass(jni, "org/webrtc/MediaCodecVideoEncoder"); |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1062 | supported_codecs_.clear(); |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1063 | |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1064 | bool is_vp8_hw_supported = jni->CallStaticBooleanMethod( |
| 1065 | j_encoder_class, |
| 1066 | GetStaticMethodID(jni, j_encoder_class, "isVp8HwSupported", "()Z")); |
| 1067 | CHECK_EXCEPTION(jni); |
| 1068 | if (is_vp8_hw_supported) { |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 1069 | ALOGD << "VP8 HW Encoder supported."; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1070 | supported_codecs_.push_back(VideoCodec(kVideoCodecVP8, "VP8", |
| 1071 | MAX_VIDEO_WIDTH, MAX_VIDEO_HEIGHT, MAX_VIDEO_FPS)); |
| 1072 | } |
| 1073 | |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 1074 | bool is_vp9_hw_supported = jni->CallStaticBooleanMethod( |
| 1075 | j_encoder_class, |
| 1076 | GetStaticMethodID(jni, j_encoder_class, "isVp9HwSupported", "()Z")); |
| 1077 | CHECK_EXCEPTION(jni); |
| 1078 | if (is_vp9_hw_supported) { |
| 1079 | ALOGD << "VP9 HW Encoder supported."; |
| 1080 | supported_codecs_.push_back(VideoCodec(kVideoCodecVP9, "VP9", |
| 1081 | MAX_VIDEO_WIDTH, MAX_VIDEO_HEIGHT, MAX_VIDEO_FPS)); |
| 1082 | } |
| 1083 | |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1084 | bool is_h264_hw_supported = jni->CallStaticBooleanMethod( |
| 1085 | j_encoder_class, |
| 1086 | GetStaticMethodID(jni, j_encoder_class, "isH264HwSupported", "()Z")); |
| 1087 | CHECK_EXCEPTION(jni); |
| 1088 | if (is_h264_hw_supported) { |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 1089 | ALOGD << "H.264 HW Encoder supported."; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1090 | supported_codecs_.push_back(VideoCodec(kVideoCodecH264, "H264", |
| 1091 | MAX_VIDEO_WIDTH, MAX_VIDEO_HEIGHT, MAX_VIDEO_FPS)); |
| 1092 | } |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1093 | } |
| 1094 | |
| 1095 | MediaCodecVideoEncoderFactory::~MediaCodecVideoEncoderFactory() {} |
| 1096 | |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 1097 | void MediaCodecVideoEncoderFactory::SetEGLContext( |
| 1098 | JNIEnv* jni, jobject render_egl_context) { |
| 1099 | ALOGD << "MediaCodecVideoEncoderFactory::SetEGLContext"; |
| 1100 | if (egl_context_) { |
| 1101 | jni->DeleteGlobalRef(egl_context_); |
| 1102 | egl_context_ = NULL; |
| 1103 | } |
| 1104 | if (!IsNull(jni, render_egl_context)) { |
| 1105 | egl_context_ = jni->NewGlobalRef(render_egl_context); |
| 1106 | if (CheckException(jni)) { |
| 1107 | ALOGE << "error calling NewGlobalRef for EGL Context."; |
| 1108 | egl_context_ = NULL; |
| 1109 | } else { |
| 1110 | jclass j_egl_context_class = |
| 1111 | FindClass(jni, "javax/microedition/khronos/egl/EGLContext"); |
| 1112 | if (!jni->IsInstanceOf(egl_context_, j_egl_context_class)) { |
| 1113 | ALOGE << "Wrong EGL Context."; |
| 1114 | jni->DeleteGlobalRef(egl_context_); |
| 1115 | egl_context_ = NULL; |
| 1116 | } |
| 1117 | } |
| 1118 | } |
| 1119 | if (egl_context_ == NULL) { |
| 1120 | ALOGW << "NULL VideoDecoder EGL context - HW surface encoding is disabled."; |
| 1121 | } |
| 1122 | } |
| 1123 | |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1124 | webrtc::VideoEncoder* MediaCodecVideoEncoderFactory::CreateVideoEncoder( |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1125 | VideoCodecType type) { |
| 1126 | if (supported_codecs_.empty()) { |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 1127 | ALOGW << "No HW video encoder for type " << (int)type; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1128 | return NULL; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1129 | } |
| 1130 | for (std::vector<VideoCodec>::const_iterator it = supported_codecs_.begin(); |
| 1131 | it != supported_codecs_.end(); ++it) { |
| 1132 | if (it->type == type) { |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 1133 | ALOGD << "Create HW video encoder for type " << (int)type << |
| 1134 | " (" << it->name << ")."; |
perkj | 30e9182 | 2015-11-20 01:31:25 -0800 | [diff] [blame] | 1135 | return new MediaCodecVideoEncoder(AttachCurrentThreadIfNeeded(), type, |
| 1136 | egl_context_); |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1137 | } |
| 1138 | } |
Alex Glaznev | ad948c4 | 2015-11-18 13:06:42 -0800 | [diff] [blame] | 1139 | ALOGW << "Can not find HW video encoder for type " << (int)type; |
glaznev@webrtc.org | b28474c | 2015-02-23 17:44:27 +0000 | [diff] [blame] | 1140 | return NULL; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1141 | } |
| 1142 | |
| 1143 | const std::vector<MediaCodecVideoEncoderFactory::VideoCodec>& |
| 1144 | MediaCodecVideoEncoderFactory::codecs() const { |
| 1145 | return supported_codecs_; |
| 1146 | } |
| 1147 | |
| 1148 | void MediaCodecVideoEncoderFactory::DestroyVideoEncoder( |
| 1149 | webrtc::VideoEncoder* encoder) { |
Alex Glaznev | fddf6e5 | 2015-10-07 16:51:02 -0700 | [diff] [blame] | 1150 | ALOGD << "Destroy video encoder."; |
glaznev@webrtc.org | 18c9247 | 2015-02-18 18:42:55 +0000 | [diff] [blame] | 1151 | delete encoder; |
| 1152 | } |
| 1153 | |
| 1154 | } // namespace webrtc_jni |
| 1155 | |