blob: a06b026c846856fa00c5c04563f3c6e18cc4f257 [file] [log] [blame]
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001/*
2 * libjingle
3 * Copyright 2015 Google Inc.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions are met:
7 *
8 * 1. Redistributions of source code must retain the above copyright notice,
9 * this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright notice,
11 * this list of conditions and the following disclaimer in the documentation
12 * and/or other materials provided with the distribution.
13 * 3. The name of the author may not be used to endorse or promote products
14 * derived from this software without specific prior written permission.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
17 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
18 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
19 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
20 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
22 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
23 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
24 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
25 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 *
27 */
28
kjellandera96e2d72016-02-04 23:52:28 -080029// NOTICE: androidmediaencoder_jni.h must be included before
30// androidmediacodeccommon.h to avoid build errors.
Henrik Kjellander15583c12016-02-10 10:53:12 +010031#include "webrtc/api/java/jni/androidmediaencoder_jni.h"
32
kjellandera96e2d72016-02-04 23:52:28 -080033#include "third_party/libyuv/include/libyuv/convert.h"
34#include "third_party/libyuv/include/libyuv/convert_from.h"
35#include "third_party/libyuv/include/libyuv/video_common.h"
Henrik Kjellander15583c12016-02-10 10:53:12 +010036#include "webrtc/api/java/jni/androidmediacodeccommon.h"
37#include "webrtc/api/java/jni/classreferenceholder.h"
38#include "webrtc/api/java/jni/native_handle_impl.h"
glaznev@webrtc.org18c92472015-02-18 18:42:55 +000039#include "webrtc/base/bind.h"
40#include "webrtc/base/checks.h"
41#include "webrtc/base/logging.h"
42#include "webrtc/base/thread.h"
perkj9576e542015-11-12 06:43:16 -080043#include "webrtc/base/thread_checker.h"
asapersson1d61a512016-01-20 01:13:46 -080044#include "webrtc/common_types.h"
Peter Boström2bc68c72015-09-24 16:22:28 +020045#include "webrtc/modules/rtp_rtcp/source/h264_bitstream_parser.h"
perkj30e91822015-11-20 01:31:25 -080046#include "webrtc/modules/video_coding/include/video_codec_interface.h"
kjellander@webrtc.orgb7ce9642015-11-18 23:04:10 +010047#include "webrtc/modules/video_coding/utility/quality_scaler.h"
48#include "webrtc/modules/video_coding/utility/vp8_header_parser.h"
Henrik Kjellander98f53512015-10-28 18:17:40 +010049#include "webrtc/system_wrappers/include/field_trial.h"
50#include "webrtc/system_wrappers/include/logcat_trace_context.h"
glaznev@webrtc.org18c92472015-02-18 18:42:55 +000051
52using rtc::Bind;
53using rtc::Thread;
54using rtc::ThreadManager;
55using rtc::scoped_ptr;
56
57using webrtc::CodecSpecificInfo;
58using webrtc::EncodedImage;
Miguel Casas-Sanchez47650702015-05-29 17:21:40 -070059using webrtc::VideoFrame;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +000060using webrtc::RTPFragmentationHeader;
61using webrtc::VideoCodec;
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +000062using webrtc::VideoCodecType;
63using webrtc::kVideoCodecH264;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +000064using webrtc::kVideoCodecVP8;
Alex Glaznevad948c42015-11-18 13:06:42 -080065using webrtc::kVideoCodecVP9;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +000066
67namespace webrtc_jni {
68
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +000069// H.264 start code length.
70#define H264_SC_LENGTH 4
71// Maximum allowed NALUs in one output frame.
72#define MAX_NALUS_PERFRAME 32
73// Maximum supported HW video encoder resolution.
74#define MAX_VIDEO_WIDTH 1280
75#define MAX_VIDEO_HEIGHT 1280
76// Maximum supported HW video encoder fps.
77#define MAX_VIDEO_FPS 30
glaznevf4decb52016-01-15 13:49:22 -080078// Maximum allowed fps value in SetRates() call.
79#define MAX_ALLOWED_VIDEO_FPS 60
80// Maximum allowed frames in encoder input queue.
81#define MAX_ENCODER_Q_SIZE 2
82// Maximum allowed latency in ms.
83#define MAX_ENCODER_LATENCY_MS 70
glaznev919ff752016-01-27 15:01:03 -080084// Maximum amount of dropped frames caused by full encoder queue - exceeding
85// this threshold means that encoder probably got stuck and need to be reset.
86#define ENCODER_STALL_FRAMEDROP_THRESHOLD 60
glaznevf4decb52016-01-15 13:49:22 -080087
88// Logging macros.
89#define TAG_ENCODER "MediaCodecVideoEncoder"
90#ifdef TRACK_BUFFER_TIMING
91#define ALOGV(...)
92 __android_log_print(ANDROID_LOG_VERBOSE, TAG_ENCODER, __VA_ARGS__)
93#else
94#define ALOGV(...)
95#endif
96#define ALOGD LOG_TAG(rtc::LS_INFO, TAG_ENCODER)
97#define ALOGW LOG_TAG(rtc::LS_WARNING, TAG_ENCODER)
98#define ALOGE LOG_TAG(rtc::LS_ERROR, TAG_ENCODER)
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +000099
asapersson1d61a512016-01-20 01:13:46 -0800100namespace {
101// Maximum time limit between incoming frames before requesting a key frame.
102const size_t kFrameDiffThresholdMs = 1100;
103const int kMinKeyFrameInterval = 2;
104} // namespace
105
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000106// MediaCodecVideoEncoder is a webrtc::VideoEncoder implementation that uses
107// Android's MediaCodec SDK API behind the scenes to implement (hopefully)
108// HW-backed video encode. This C++ class is implemented as a very thin shim,
109// delegating all of the interesting work to org.webrtc.MediaCodecVideoEncoder.
110// MediaCodecVideoEncoder is created, operated, and destroyed on a single
111// thread, currently the libjingle Worker thread.
112class MediaCodecVideoEncoder : public webrtc::VideoEncoder,
113 public rtc::MessageHandler {
114 public:
115 virtual ~MediaCodecVideoEncoder();
perkj9576e542015-11-12 06:43:16 -0800116 MediaCodecVideoEncoder(JNIEnv* jni,
perkj30e91822015-11-20 01:31:25 -0800117 VideoCodecType codecType,
118 jobject egl_context);
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000119
120 // webrtc::VideoEncoder implementation. Everything trampolines to
121 // |codec_thread_| for execution.
122 int32_t InitEncode(const webrtc::VideoCodec* codec_settings,
123 int32_t /* number_of_cores */,
124 size_t /* max_payload_size */) override;
pbos22993e12015-10-19 02:39:06 -0700125 int32_t Encode(const webrtc::VideoFrame& input_image,
126 const webrtc::CodecSpecificInfo* /* codec_specific_info */,
127 const std::vector<webrtc::FrameType>* frame_types) override;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000128 int32_t RegisterEncodeCompleteCallback(
129 webrtc::EncodedImageCallback* callback) override;
130 int32_t Release() override;
131 int32_t SetChannelParameters(uint32_t /* packet_loss */,
132 int64_t /* rtt */) override;
133 int32_t SetRates(uint32_t new_bit_rate, uint32_t frame_rate) override;
134
135 // rtc::MessageHandler implementation.
136 void OnMessage(rtc::Message* msg) override;
137
jackychen61b4d512015-04-21 15:30:11 -0700138 void OnDroppedFrame() override;
139
jackychen6e2ce6e2015-07-13 16:26:33 -0700140 int GetTargetFramerate() override;
141
Perec2922f2016-01-27 15:25:46 +0100142 bool SupportsNativeHandle() const override { return egl_context_ != nullptr; }
Peter Boströmb7d9a972015-12-18 16:01:11 +0100143 const char* ImplementationName() const override;
144
145 private:
146 // CHECK-fail if not running on |codec_thread_|.
147 void CheckOnCodecThread();
perkj30e91822015-11-20 01:31:25 -0800148
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000149 private:
perkj9576e542015-11-12 06:43:16 -0800150 // ResetCodecOnCodecThread() calls ReleaseOnCodecThread() and
151 // InitEncodeOnCodecThread() in an attempt to restore the codec to an
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000152 // operable state. Necessary after all manner of OMX-layer errors.
perkj9576e542015-11-12 06:43:16 -0800153 bool ResetCodecOnCodecThread();
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000154
155 // Implementation of webrtc::VideoEncoder methods above, all running on the
156 // codec thread exclusively.
157 //
158 // If width==0 then this is assumed to be a re-initialization and the
159 // previously-current values are reused instead of the passed parameters
160 // (makes it easier to reason about thread-safety).
perkj30e91822015-11-20 01:31:25 -0800161 int32_t InitEncodeOnCodecThread(int width, int height, int kbps, int fps,
162 bool use_surface);
163 // Reconfigure to match |frame| in width, height. Also reconfigures the
164 // encoder if |frame| is a texture/byte buffer and the encoder is initialized
165 // for byte buffer/texture. Returns false if reconfiguring fails.
perkj9576e542015-11-12 06:43:16 -0800166 bool MaybeReconfigureEncoderOnCodecThread(const webrtc::VideoFrame& frame);
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000167 int32_t EncodeOnCodecThread(
Miguel Casas-Sanchez47650702015-05-29 17:21:40 -0700168 const webrtc::VideoFrame& input_image,
pbos22993e12015-10-19 02:39:06 -0700169 const std::vector<webrtc::FrameType>* frame_types);
perkj9576e542015-11-12 06:43:16 -0800170 bool EncodeByteBufferOnCodecThread(JNIEnv* jni,
171 bool key_frame, const webrtc::VideoFrame& frame, int input_buffer_index);
perkj30e91822015-11-20 01:31:25 -0800172 bool EncodeTextureOnCodecThread(JNIEnv* jni,
173 bool key_frame, const webrtc::VideoFrame& frame);
perkj9576e542015-11-12 06:43:16 -0800174
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000175 int32_t RegisterEncodeCompleteCallbackOnCodecThread(
176 webrtc::EncodedImageCallback* callback);
177 int32_t ReleaseOnCodecThread();
178 int32_t SetRatesOnCodecThread(uint32_t new_bit_rate, uint32_t frame_rate);
179
180 // Helper accessors for MediaCodecVideoEncoder$OutputBufferInfo members.
181 int GetOutputBufferInfoIndex(JNIEnv* jni, jobject j_output_buffer_info);
182 jobject GetOutputBufferInfoBuffer(JNIEnv* jni, jobject j_output_buffer_info);
183 bool GetOutputBufferInfoIsKeyFrame(JNIEnv* jni, jobject j_output_buffer_info);
184 jlong GetOutputBufferInfoPresentationTimestampUs(
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000185 JNIEnv* jni, jobject j_output_buffer_info);
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000186
187 // Deliver any outputs pending in the MediaCodec to our |callback_| and return
188 // true on success.
189 bool DeliverPendingOutputs(JNIEnv* jni);
190
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000191 // Search for H.264 start codes.
192 int32_t NextNaluPosition(uint8_t *buffer, size_t buffer_size);
193
glaznev94291482016-02-01 13:17:18 -0800194 // Displays encoder statistics.
195 void LogStatistics(bool force_log);
196
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000197 // Type of video codec.
198 VideoCodecType codecType_;
199
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000200 // Valid all the time since RegisterEncodeCompleteCallback() Invoke()s to
201 // |codec_thread_| synchronously.
202 webrtc::EncodedImageCallback* callback_;
203
204 // State that is constant for the lifetime of this object once the ctor
205 // returns.
206 scoped_ptr<Thread> codec_thread_; // Thread on which to operate MediaCodec.
perkj9576e542015-11-12 06:43:16 -0800207 rtc::ThreadChecker codec_thread_checker_;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000208 ScopedGlobalRef<jclass> j_media_codec_video_encoder_class_;
209 ScopedGlobalRef<jobject> j_media_codec_video_encoder_;
210 jmethodID j_init_encode_method_;
perkj9576e542015-11-12 06:43:16 -0800211 jmethodID j_get_input_buffers_method_;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000212 jmethodID j_dequeue_input_buffer_method_;
perkj9576e542015-11-12 06:43:16 -0800213 jmethodID j_encode_buffer_method_;
perkj30e91822015-11-20 01:31:25 -0800214 jmethodID j_encode_texture_method_;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000215 jmethodID j_release_method_;
216 jmethodID j_set_rates_method_;
217 jmethodID j_dequeue_output_buffer_method_;
218 jmethodID j_release_output_buffer_method_;
219 jfieldID j_color_format_field_;
220 jfieldID j_info_index_field_;
221 jfieldID j_info_buffer_field_;
222 jfieldID j_info_is_key_frame_field_;
223 jfieldID j_info_presentation_timestamp_us_field_;
224
225 // State that is valid only between InitEncode() and the next Release().
226 // Touched only on codec_thread_ so no explicit synchronization necessary.
227 int width_; // Frame width in pixels.
228 int height_; // Frame height in pixels.
229 bool inited_;
perkj30e91822015-11-20 01:31:25 -0800230 bool use_surface_;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000231 uint16_t picture_id_;
232 enum libyuv::FourCC encoder_fourcc_; // Encoder color space format.
233 int last_set_bitrate_kbps_; // Last-requested bitrate in kbps.
234 int last_set_fps_; // Last-requested frame rate.
235 int64_t current_timestamp_us_; // Current frame timestamps in us.
236 int frames_received_; // Number of frames received by encoder.
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000237 int frames_encoded_; // Number of frames encoded by encoder.
glaznev919ff752016-01-27 15:01:03 -0800238 int frames_dropped_media_encoder_; // Number of frames dropped by encoder.
239 // Number of dropped frames caused by full queue.
240 int consecutive_full_queue_frame_drops_;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000241 int frames_in_queue_; // Number of frames in encoder queue.
glaznev94291482016-02-01 13:17:18 -0800242 int64_t stat_start_time_ms_; // Start time for statistics.
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000243 int current_frames_; // Number of frames in the current statistics interval.
244 int current_bytes_; // Encoded bytes in the current statistics interval.
glaznevf4decb52016-01-15 13:49:22 -0800245 int current_acc_qp_; // Accumulated QP in the current statistics interval.
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000246 int current_encoding_time_ms_; // Overall encoding time in the current second
247 int64_t last_input_timestamp_ms_; // Timestamp of last received yuv frame.
248 int64_t last_output_timestamp_ms_; // Timestamp of last encoded frame.
249 std::vector<int32_t> timestamps_; // Video frames timestamp queue.
250 std::vector<int64_t> render_times_ms_; // Video frames render time queue.
251 std::vector<int64_t> frame_rtc_times_ms_; // Time when video frame is sent to
252 // encoder input.
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000253 int32_t output_timestamp_; // Last output frame timestamp from timestamps_ Q.
254 int64_t output_render_time_ms_; // Last output frame render time from
255 // render_times_ms_ queue.
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000256 // Frame size in bytes fed to MediaCodec.
257 int yuv_size_;
258 // True only when between a callback_->Encoded() call return a positive value
259 // and the next Encode() call being ignored.
260 bool drop_next_input_frame_;
261 // Global references; must be deleted in Release().
262 std::vector<jobject> input_buffers_;
Peter Boström2bc68c72015-09-24 16:22:28 +0200263 webrtc::QualityScaler quality_scaler_;
jackychen61b4d512015-04-21 15:30:11 -0700264 // Dynamic resolution change, off by default.
265 bool scale_;
Peter Boström2bc68c72015-09-24 16:22:28 +0200266
267 // H264 bitstream parser, used to extract QP from encoded bitstreams.
268 webrtc::H264BitstreamParser h264_bitstream_parser_;
Alex Glaznevad948c42015-11-18 13:06:42 -0800269
270 // VP9 variables to populate codec specific structure.
271 webrtc::GofInfoVP9 gof_; // Contains each frame's temporal information for
272 // non-flexible VP9 mode.
273 uint8_t tl0_pic_idx_;
274 size_t gof_idx_;
perkj30e91822015-11-20 01:31:25 -0800275
276 // EGL context - owned by factory, should not be allocated/destroyed
277 // by MediaCodecVideoEncoder.
278 jobject egl_context_;
asapersson1d61a512016-01-20 01:13:46 -0800279
280 // Temporary fix for VP8.
281 // Sends a key frame if frames are largely spaced apart (possibly
282 // corresponding to a large image change).
283 int64_t last_frame_received_ms_;
284 int frames_received_since_last_key_;
285 webrtc::VideoCodecMode codec_mode_;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000286};
287
288MediaCodecVideoEncoder::~MediaCodecVideoEncoder() {
289 // Call Release() to ensure no more callbacks to us after we are deleted.
290 Release();
291}
292
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000293MediaCodecVideoEncoder::MediaCodecVideoEncoder(
perkj30e91822015-11-20 01:31:25 -0800294 JNIEnv* jni, VideoCodecType codecType, jobject egl_context) :
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000295 codecType_(codecType),
296 callback_(NULL),
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000297 codec_thread_(new Thread()),
298 j_media_codec_video_encoder_class_(
299 jni,
300 FindClass(jni, "org/webrtc/MediaCodecVideoEncoder")),
301 j_media_codec_video_encoder_(
302 jni,
303 jni->NewObject(*j_media_codec_video_encoder_class_,
304 GetMethodID(jni,
305 *j_media_codec_video_encoder_class_,
306 "<init>",
perkj30e91822015-11-20 01:31:25 -0800307 "()V"))),
kjellander60ca31b2016-01-04 10:15:53 -0800308 inited_(false),
309 use_surface_(false),
310 picture_id_(0),
perkj30e91822015-11-20 01:31:25 -0800311 egl_context_(egl_context) {
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000312 ScopedLocalRefFrame local_ref_frame(jni);
313 // It would be nice to avoid spinning up a new thread per MediaCodec, and
314 // instead re-use e.g. the PeerConnectionFactory's |worker_thread_|, but bug
315 // 2732 means that deadlocks abound. This class synchronously trampolines
316 // to |codec_thread_|, so if anything else can be coming to _us_ from
317 // |codec_thread_|, or from any thread holding the |_sendCritSect| described
318 // in the bug, we have a problem. For now work around that with a dedicated
319 // thread.
320 codec_thread_->SetName("MediaCodecVideoEncoder", NULL);
henrikg91d6ede2015-09-17 00:24:34 -0700321 RTC_CHECK(codec_thread_->Start()) << "Failed to start MediaCodecVideoEncoder";
perkj9576e542015-11-12 06:43:16 -0800322 codec_thread_checker_.DetachFromThread();
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000323 jclass j_output_buffer_info_class =
324 FindClass(jni, "org/webrtc/MediaCodecVideoEncoder$OutputBufferInfo");
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000325 j_init_encode_method_ = GetMethodID(
326 jni,
327 *j_media_codec_video_encoder_class_,
328 "initEncode",
perkj30e91822015-11-20 01:31:25 -0800329 "(Lorg/webrtc/MediaCodecVideoEncoder$VideoCodecType;"
perkj48477c12015-12-18 00:34:37 -0800330 "IIIILorg/webrtc/EglBase14$Context;)Z");
perkj9576e542015-11-12 06:43:16 -0800331 j_get_input_buffers_method_ = GetMethodID(
332 jni,
333 *j_media_codec_video_encoder_class_,
334 "getInputBuffers",
335 "()[Ljava/nio/ByteBuffer;");
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000336 j_dequeue_input_buffer_method_ = GetMethodID(
337 jni, *j_media_codec_video_encoder_class_, "dequeueInputBuffer", "()I");
perkj9576e542015-11-12 06:43:16 -0800338 j_encode_buffer_method_ = GetMethodID(
339 jni, *j_media_codec_video_encoder_class_, "encodeBuffer", "(ZIIJ)Z");
perkj30e91822015-11-20 01:31:25 -0800340 j_encode_texture_method_ = GetMethodID(
341 jni, *j_media_codec_video_encoder_class_, "encodeTexture",
342 "(ZI[FJ)Z");
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000343 j_release_method_ =
344 GetMethodID(jni, *j_media_codec_video_encoder_class_, "release", "()V");
345 j_set_rates_method_ = GetMethodID(
346 jni, *j_media_codec_video_encoder_class_, "setRates", "(II)Z");
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000347 j_dequeue_output_buffer_method_ = GetMethodID(
348 jni,
349 *j_media_codec_video_encoder_class_,
350 "dequeueOutputBuffer",
351 "()Lorg/webrtc/MediaCodecVideoEncoder$OutputBufferInfo;");
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000352 j_release_output_buffer_method_ = GetMethodID(
353 jni, *j_media_codec_video_encoder_class_, "releaseOutputBuffer", "(I)Z");
354
355 j_color_format_field_ =
356 GetFieldID(jni, *j_media_codec_video_encoder_class_, "colorFormat", "I");
357 j_info_index_field_ =
358 GetFieldID(jni, j_output_buffer_info_class, "index", "I");
359 j_info_buffer_field_ = GetFieldID(
360 jni, j_output_buffer_info_class, "buffer", "Ljava/nio/ByteBuffer;");
361 j_info_is_key_frame_field_ =
362 GetFieldID(jni, j_output_buffer_info_class, "isKeyFrame", "Z");
363 j_info_presentation_timestamp_us_field_ = GetFieldID(
364 jni, j_output_buffer_info_class, "presentationTimestampUs", "J");
365 CHECK_EXCEPTION(jni) << "MediaCodecVideoEncoder ctor failed";
Alex Glaznevad948c42015-11-18 13:06:42 -0800366 srand(time(NULL));
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000367 AllowBlockingCalls();
368}
369
370int32_t MediaCodecVideoEncoder::InitEncode(
371 const webrtc::VideoCodec* codec_settings,
372 int32_t /* number_of_cores */,
373 size_t /* max_payload_size */) {
jackychen61b4d512015-04-21 15:30:11 -0700374 const int kMinWidth = 320;
375 const int kMinHeight = 180;
jackychen98d8cf52015-05-21 11:12:02 -0700376 const int kLowQpThresholdDenominator = 3;
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000377 if (codec_settings == NULL) {
Alex Glaznevfddf6e52015-10-07 16:51:02 -0700378 ALOGE << "NULL VideoCodec instance";
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000379 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
380 }
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000381 // Factory should guard against other codecs being used with us.
henrikg91d6ede2015-09-17 00:24:34 -0700382 RTC_CHECK(codec_settings->codecType == codecType_)
383 << "Unsupported codec " << codec_settings->codecType << " for "
384 << codecType_;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000385
Alex Glaznevfddf6e52015-10-07 16:51:02 -0700386 ALOGD << "InitEncode request";
asapersson1d61a512016-01-20 01:13:46 -0800387 codec_mode_ = codec_settings->mode;
Alex Glaznevad948c42015-11-18 13:06:42 -0800388 scale_ = (codecType_ != kVideoCodecVP9) && (webrtc::field_trial::FindFullName(
389 "WebRTC-MediaCodecVideoEncoder-AutomaticResize") == "Enabled");
Alex Glaznevfddf6e52015-10-07 16:51:02 -0700390 ALOGD << "Encoder automatic resize " << (scale_ ? "enabled" : "disabled");
Peter Boström2bc68c72015-09-24 16:22:28 +0200391 if (scale_) {
392 if (codecType_ == kVideoCodecVP8) {
393 // QP is obtained from VP8-bitstream for HW, so the QP corresponds to the
394 // (internal) range: [0, 127]. And we cannot change QP_max in HW, so it is
395 // always = 127. Note that in SW, QP is that of the user-level range [0,
396 // 63].
397 const int kMaxQp = 127;
glaznev919ff752016-01-27 15:01:03 -0800398 const int kBadQpThreshold = 95;
399 quality_scaler_.Init(
400 kMaxQp / kLowQpThresholdDenominator, kBadQpThreshold, false);
Peter Boström2bc68c72015-09-24 16:22:28 +0200401 } else if (codecType_ == kVideoCodecH264) {
402 // H264 QP is in the range [0, 51].
403 const int kMaxQp = 51;
Peter Boström17417702015-09-25 17:03:26 +0200404 const int kBadQpThreshold = 40;
glaznev919ff752016-01-27 15:01:03 -0800405 quality_scaler_.Init(
406 kMaxQp / kLowQpThresholdDenominator, kBadQpThreshold, false);
Peter Boström2bc68c72015-09-24 16:22:28 +0200407 } else {
408 // When adding codec support to additional hardware codecs, also configure
409 // their QP thresholds for scaling.
410 RTC_NOTREACHED() << "Unsupported codec without configured QP thresholds.";
411 }
412 quality_scaler_.SetMinResolution(kMinWidth, kMinHeight);
413 quality_scaler_.ReportFramerate(codec_settings->maxFramerate);
jackychen61b4d512015-04-21 15:30:11 -0700414 }
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000415 return codec_thread_->Invoke<int32_t>(
416 Bind(&MediaCodecVideoEncoder::InitEncodeOnCodecThread,
417 this,
418 codec_settings->width,
419 codec_settings->height,
420 codec_settings->startBitrate,
perkj30e91822015-11-20 01:31:25 -0800421 codec_settings->maxFramerate,
422 false /* use_surface */));
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000423}
424
425int32_t MediaCodecVideoEncoder::Encode(
Miguel Casas-Sanchez47650702015-05-29 17:21:40 -0700426 const webrtc::VideoFrame& frame,
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000427 const webrtc::CodecSpecificInfo* /* codec_specific_info */,
pbos22993e12015-10-19 02:39:06 -0700428 const std::vector<webrtc::FrameType>* frame_types) {
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000429 return codec_thread_->Invoke<int32_t>(Bind(
430 &MediaCodecVideoEncoder::EncodeOnCodecThread, this, frame, frame_types));
431}
432
433int32_t MediaCodecVideoEncoder::RegisterEncodeCompleteCallback(
434 webrtc::EncodedImageCallback* callback) {
435 return codec_thread_->Invoke<int32_t>(
436 Bind(&MediaCodecVideoEncoder::RegisterEncodeCompleteCallbackOnCodecThread,
437 this,
438 callback));
439}
440
441int32_t MediaCodecVideoEncoder::Release() {
Alex Glaznevfddf6e52015-10-07 16:51:02 -0700442 ALOGD << "EncoderRelease request";
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000443 return codec_thread_->Invoke<int32_t>(
444 Bind(&MediaCodecVideoEncoder::ReleaseOnCodecThread, this));
445}
446
447int32_t MediaCodecVideoEncoder::SetChannelParameters(uint32_t /* packet_loss */,
448 int64_t /* rtt */) {
449 return WEBRTC_VIDEO_CODEC_OK;
450}
451
452int32_t MediaCodecVideoEncoder::SetRates(uint32_t new_bit_rate,
453 uint32_t frame_rate) {
454 return codec_thread_->Invoke<int32_t>(
455 Bind(&MediaCodecVideoEncoder::SetRatesOnCodecThread,
456 this,
457 new_bit_rate,
458 frame_rate));
459}
460
461void MediaCodecVideoEncoder::OnMessage(rtc::Message* msg) {
perkj9576e542015-11-12 06:43:16 -0800462 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000463 JNIEnv* jni = AttachCurrentThreadIfNeeded();
464 ScopedLocalRefFrame local_ref_frame(jni);
465
466 // We only ever send one message to |this| directly (not through a Bind()'d
467 // functor), so expect no ID/data.
henrikg91d6ede2015-09-17 00:24:34 -0700468 RTC_CHECK(!msg->message_id) << "Unexpected message!";
469 RTC_CHECK(!msg->pdata) << "Unexpected message!";
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000470 if (!inited_) {
471 return;
472 }
473
474 // It would be nice to recover from a failure here if one happened, but it's
475 // unclear how to signal such a failure to the app, so instead we stay silent
476 // about it and let the next app-called API method reveal the borkedness.
477 DeliverPendingOutputs(jni);
478 codec_thread_->PostDelayed(kMediaCodecPollMs, this);
479}
480
perkj9576e542015-11-12 06:43:16 -0800481bool MediaCodecVideoEncoder::ResetCodecOnCodecThread() {
482 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
483 ALOGE << "ResetOnCodecThread";
484 if (ReleaseOnCodecThread() != WEBRTC_VIDEO_CODEC_OK ||
perkj30e91822015-11-20 01:31:25 -0800485 InitEncodeOnCodecThread(width_, height_, 0, 0, false) !=
486 WEBRTC_VIDEO_CODEC_OK) {
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000487 // TODO(fischman): wouldn't it be nice if there was a way to gracefully
488 // degrade to a SW encoder at this point? There isn't one AFAICT :(
489 // https://code.google.com/p/webrtc/issues/detail?id=2920
perkj9576e542015-11-12 06:43:16 -0800490 return false;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000491 }
perkj9576e542015-11-12 06:43:16 -0800492 return true;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000493}
494
495int32_t MediaCodecVideoEncoder::InitEncodeOnCodecThread(
perkj30e91822015-11-20 01:31:25 -0800496 int width, int height, int kbps, int fps, bool use_surface) {
perkj9576e542015-11-12 06:43:16 -0800497 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
perkj30e91822015-11-20 01:31:25 -0800498 RTC_CHECK(!use_surface || egl_context_ != nullptr) << "EGL context not set.";
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000499 JNIEnv* jni = AttachCurrentThreadIfNeeded();
500 ScopedLocalRefFrame local_ref_frame(jni);
501
Alex Glaznevfddf6e52015-10-07 16:51:02 -0700502 ALOGD << "InitEncodeOnCodecThread Type: " << (int)codecType_ << ", " <<
503 width << " x " << height << ". Bitrate: " << kbps <<
504 " kbps. Fps: " << fps;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000505 if (kbps == 0) {
506 kbps = last_set_bitrate_kbps_;
507 }
508 if (fps == 0) {
glaznevf4decb52016-01-15 13:49:22 -0800509 fps = MAX_VIDEO_FPS;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000510 }
511
512 width_ = width;
513 height_ = height;
514 last_set_bitrate_kbps_ = kbps;
glaznevf4decb52016-01-15 13:49:22 -0800515 last_set_fps_ = (fps < MAX_VIDEO_FPS) ? fps : MAX_VIDEO_FPS;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000516 yuv_size_ = width_ * height_ * 3 / 2;
517 frames_received_ = 0;
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000518 frames_encoded_ = 0;
glaznev919ff752016-01-27 15:01:03 -0800519 frames_dropped_media_encoder_ = 0;
520 consecutive_full_queue_frame_drops_ = 0;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000521 frames_in_queue_ = 0;
522 current_timestamp_us_ = 0;
glaznev94291482016-02-01 13:17:18 -0800523 stat_start_time_ms_ = GetCurrentTimeMs();
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000524 current_frames_ = 0;
525 current_bytes_ = 0;
glaznevf4decb52016-01-15 13:49:22 -0800526 current_acc_qp_ = 0;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000527 current_encoding_time_ms_ = 0;
528 last_input_timestamp_ms_ = -1;
529 last_output_timestamp_ms_ = -1;
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000530 output_timestamp_ = 0;
531 output_render_time_ms_ = 0;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000532 timestamps_.clear();
533 render_times_ms_.clear();
534 frame_rtc_times_ms_.clear();
535 drop_next_input_frame_ = false;
perkj30e91822015-11-20 01:31:25 -0800536 use_surface_ = use_surface;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000537 picture_id_ = static_cast<uint16_t>(rand()) & 0x7FFF;
Alex Glaznevad948c42015-11-18 13:06:42 -0800538 gof_.SetGofInfoVP9(webrtc::TemporalStructureMode::kTemporalStructureMode1);
539 tl0_pic_idx_ = static_cast<uint8_t>(rand());
540 gof_idx_ = 0;
asapersson1d61a512016-01-20 01:13:46 -0800541 last_frame_received_ms_ = -1;
542 frames_received_since_last_key_ = kMinKeyFrameInterval;
perkj9576e542015-11-12 06:43:16 -0800543
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000544 // We enforce no extra stride/padding in the format creation step.
Perec2922f2016-01-27 15:25:46 +0100545 jobject j_video_codec_enum = JavaEnumFromIndexAndClassName(
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000546 jni, "MediaCodecVideoEncoder$VideoCodecType", codecType_);
perkj9576e542015-11-12 06:43:16 -0800547 const bool encode_status = jni->CallBooleanMethod(
548 *j_media_codec_video_encoder_, j_init_encode_method_,
perkj30e91822015-11-20 01:31:25 -0800549 j_video_codec_enum, width, height, kbps, fps,
550 (use_surface ? egl_context_ : nullptr));
perkj9576e542015-11-12 06:43:16 -0800551 if (!encode_status) {
552 ALOGE << "Failed to configure encoder.";
553 return WEBRTC_VIDEO_CODEC_ERROR;
554 }
555 CHECK_EXCEPTION(jni);
556
Per598242a2015-11-26 14:28:55 +0100557 if (!use_surface) {
perkj30e91822015-11-20 01:31:25 -0800558 jobjectArray input_buffers = reinterpret_cast<jobjectArray>(
559 jni->CallObjectMethod(*j_media_codec_video_encoder_,
560 j_get_input_buffers_method_));
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000561 CHECK_EXCEPTION(jni);
perkj30e91822015-11-20 01:31:25 -0800562 if (IsNull(jni, input_buffers)) {
563 return WEBRTC_VIDEO_CODEC_ERROR;
564 }
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000565
perkj30e91822015-11-20 01:31:25 -0800566 switch (GetIntField(jni, *j_media_codec_video_encoder_,
567 j_color_format_field_)) {
568 case COLOR_FormatYUV420Planar:
569 encoder_fourcc_ = libyuv::FOURCC_YU12;
570 break;
571 case COLOR_FormatYUV420SemiPlanar:
572 case COLOR_QCOM_FormatYUV420SemiPlanar:
573 case COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m:
574 encoder_fourcc_ = libyuv::FOURCC_NV12;
575 break;
576 default:
577 LOG(LS_ERROR) << "Wrong color format.";
578 return WEBRTC_VIDEO_CODEC_ERROR;
579 }
580 size_t num_input_buffers = jni->GetArrayLength(input_buffers);
581 RTC_CHECK(input_buffers_.empty())
582 << "Unexpected double InitEncode without Release";
583 input_buffers_.resize(num_input_buffers);
584 for (size_t i = 0; i < num_input_buffers; ++i) {
585 input_buffers_[i] =
586 jni->NewGlobalRef(jni->GetObjectArrayElement(input_buffers, i));
587 int64_t yuv_buffer_capacity =
588 jni->GetDirectBufferCapacity(input_buffers_[i]);
589 CHECK_EXCEPTION(jni);
590 RTC_CHECK(yuv_buffer_capacity >= yuv_size_) << "Insufficient capacity";
591 }
592 }
perkj9576e542015-11-12 06:43:16 -0800593
594 inited_ = true;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000595 codec_thread_->PostDelayed(kMediaCodecPollMs, this);
596 return WEBRTC_VIDEO_CODEC_OK;
597}
598
599int32_t MediaCodecVideoEncoder::EncodeOnCodecThread(
Miguel Casas-Sanchez47650702015-05-29 17:21:40 -0700600 const webrtc::VideoFrame& frame,
pbos22993e12015-10-19 02:39:06 -0700601 const std::vector<webrtc::FrameType>* frame_types) {
perkj9576e542015-11-12 06:43:16 -0800602 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000603 JNIEnv* jni = AttachCurrentThreadIfNeeded();
604 ScopedLocalRefFrame local_ref_frame(jni);
605
606 if (!inited_) {
607 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
608 }
perkj9576e542015-11-12 06:43:16 -0800609
asapersson1d61a512016-01-20 01:13:46 -0800610 bool send_key_frame = false;
glaznev94291482016-02-01 13:17:18 -0800611 if (codec_mode_ == webrtc::kRealtimeVideo) {
asapersson1d61a512016-01-20 01:13:46 -0800612 ++frames_received_since_last_key_;
613 int64_t now_ms = GetCurrentTimeMs();
614 if (last_frame_received_ms_ != -1 &&
615 (now_ms - last_frame_received_ms_) > kFrameDiffThresholdMs) {
616 // Add limit to prevent triggering a key for every frame for very low
617 // framerates (e.g. if frame diff > kFrameDiffThresholdMs).
618 if (frames_received_since_last_key_ > kMinKeyFrameInterval) {
619 ALOGD << "Send key, frame diff: " << (now_ms - last_frame_received_ms_);
620 send_key_frame = true;
621 }
622 frames_received_since_last_key_ = 0;
623 }
624 last_frame_received_ms_ = now_ms;
625 }
626
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000627 frames_received_++;
628 if (!DeliverPendingOutputs(jni)) {
perkj9576e542015-11-12 06:43:16 -0800629 if (!ResetCodecOnCodecThread())
630 return WEBRTC_VIDEO_CODEC_ERROR;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000631 }
glaznevf4decb52016-01-15 13:49:22 -0800632 if (frames_encoded_ < kMaxEncodedLogFrames) {
glaznev94291482016-02-01 13:17:18 -0800633 ALOGD << "Encoder frame in # " << (frames_received_ - 1) <<
634 ". TS: " << (int)(current_timestamp_us_ / 1000) <<
635 ". Q: " << frames_in_queue_ <<
636 ". Fps: " << last_set_fps_ <<
637 ". Kbps: " << last_set_bitrate_kbps_;
glaznevf4decb52016-01-15 13:49:22 -0800638 }
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000639
640 if (drop_next_input_frame_) {
perkj9576e542015-11-12 06:43:16 -0800641 ALOGW << "Encoder drop frame - failed callback.";
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000642 drop_next_input_frame_ = false;
glaznevf4decb52016-01-15 13:49:22 -0800643 current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
glaznev919ff752016-01-27 15:01:03 -0800644 frames_dropped_media_encoder_++;
glaznevf4decb52016-01-15 13:49:22 -0800645 OnDroppedFrame();
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000646 return WEBRTC_VIDEO_CODEC_OK;
647 }
648
henrikg91d6ede2015-09-17 00:24:34 -0700649 RTC_CHECK(frame_types->size() == 1) << "Unexpected stream count";
Peter Boström2bc68c72015-09-24 16:22:28 +0200650
glaznevf4decb52016-01-15 13:49:22 -0800651 // Check if we accumulated too many frames in encoder input buffers
652 // or the encoder latency exceeds 70 ms and drop frame if so.
653 if (frames_in_queue_ > 0 && last_input_timestamp_ms_ >= 0) {
654 int encoder_latency_ms = last_input_timestamp_ms_ -
655 last_output_timestamp_ms_;
656 if (frames_in_queue_ > MAX_ENCODER_Q_SIZE ||
657 encoder_latency_ms > MAX_ENCODER_LATENCY_MS) {
658 ALOGD << "Drop frame - encoder is behind by " << encoder_latency_ms <<
glaznev94291482016-02-01 13:17:18 -0800659 " ms. Q size: " << frames_in_queue_ << ". TS: " <<
660 (int)(current_timestamp_us_ / 1000) << ". Fps: " << last_set_fps_ <<
661 ". Consecutive drops: " << consecutive_full_queue_frame_drops_ ;
glaznevf4decb52016-01-15 13:49:22 -0800662 current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
glaznev919ff752016-01-27 15:01:03 -0800663 consecutive_full_queue_frame_drops_++;
664 if (consecutive_full_queue_frame_drops_ >=
665 ENCODER_STALL_FRAMEDROP_THRESHOLD) {
666 ALOGE << "Encoder got stuck. Reset.";
667 ResetCodecOnCodecThread();
668 return WEBRTC_VIDEO_CODEC_ERROR;
669 }
670 frames_dropped_media_encoder_++;
glaznevf4decb52016-01-15 13:49:22 -0800671 OnDroppedFrame();
672 return WEBRTC_VIDEO_CODEC_OK;
673 }
674 }
glaznev919ff752016-01-27 15:01:03 -0800675 consecutive_full_queue_frame_drops_ = 0;
glaznevf4decb52016-01-15 13:49:22 -0800676
Per598242a2015-11-26 14:28:55 +0100677 VideoFrame input_frame = frame;
678 if (scale_) {
679 // Check framerate before spatial resolution change.
680 quality_scaler_.OnEncodeFrame(frame);
681 const webrtc::QualityScaler::Resolution scaled_resolution =
682 quality_scaler_.GetScaledResolution();
683 if (scaled_resolution.width != frame.width() ||
684 scaled_resolution.height != frame.height()) {
685 if (frame.native_handle() != nullptr) {
686 rtc::scoped_refptr<webrtc::VideoFrameBuffer> scaled_buffer(
687 static_cast<AndroidTextureBuffer*>(
Per71f5a9a2015-12-11 09:32:37 +0100688 frame.video_frame_buffer().get())->ScaleAndRotate(
Per598242a2015-11-26 14:28:55 +0100689 scaled_resolution.width,
Per71f5a9a2015-12-11 09:32:37 +0100690 scaled_resolution.height,
691 webrtc::kVideoRotation_0));
Per598242a2015-11-26 14:28:55 +0100692 input_frame.set_video_frame_buffer(scaled_buffer);
693 } else {
694 input_frame = quality_scaler_.GetScaledFrame(frame);
695 }
696 }
697 }
jackychen61b4d512015-04-21 15:30:11 -0700698
perkj9576e542015-11-12 06:43:16 -0800699 if (!MaybeReconfigureEncoderOnCodecThread(input_frame)) {
700 ALOGE << "Failed to reconfigure encoder.";
701 return WEBRTC_VIDEO_CODEC_ERROR;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000702 }
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000703
glaznevf4decb52016-01-15 13:49:22 -0800704 // Save time when input frame is sent to the encoder input.
705 frame_rtc_times_ms_.push_back(GetCurrentTimeMs());
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000706
asapersson1d61a512016-01-20 01:13:46 -0800707 const bool key_frame =
708 frame_types->front() != webrtc::kVideoFrameDelta || send_key_frame;
perkj30e91822015-11-20 01:31:25 -0800709 bool encode_status = true;
710 if (!input_frame.native_handle()) {
711 int j_input_buffer_index = jni->CallIntMethod(*j_media_codec_video_encoder_,
712 j_dequeue_input_buffer_method_);
713 CHECK_EXCEPTION(jni);
714 if (j_input_buffer_index == -1) {
715 // Video codec falls behind - no input buffer available.
716 ALOGW << "Encoder drop frame - no input buffers available";
glaznevf4decb52016-01-15 13:49:22 -0800717 frame_rtc_times_ms_.erase(frame_rtc_times_ms_.begin());
glaznev919ff752016-01-27 15:01:03 -0800718 current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
719 frames_dropped_media_encoder_++;
720 OnDroppedFrame();
perkj30e91822015-11-20 01:31:25 -0800721 return WEBRTC_VIDEO_CODEC_OK; // TODO(fischman): see webrtc bug 2887.
722 }
723 if (j_input_buffer_index == -2) {
724 ResetCodecOnCodecThread();
725 return WEBRTC_VIDEO_CODEC_ERROR;
726 }
727 encode_status = EncodeByteBufferOnCodecThread(jni, key_frame, input_frame,
728 j_input_buffer_index);
729 } else {
730 encode_status = EncodeTextureOnCodecThread(jni, key_frame, input_frame);
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000731 }
perkj30e91822015-11-20 01:31:25 -0800732
733 if (!encode_status) {
734 ALOGE << "Failed encode frame with timestamp: " << input_frame.timestamp();
perkj9576e542015-11-12 06:43:16 -0800735 ResetCodecOnCodecThread();
perkj12f68022015-10-16 13:31:45 +0200736 return WEBRTC_VIDEO_CODEC_ERROR;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000737 }
738
perkj9576e542015-11-12 06:43:16 -0800739 last_input_timestamp_ms_ =
740 current_timestamp_us_ / rtc::kNumMicrosecsPerMillisec;
perkj12f68022015-10-16 13:31:45 +0200741 frames_in_queue_++;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000742
perkj12f68022015-10-16 13:31:45 +0200743 // Save input image timestamps for later output
744 timestamps_.push_back(input_frame.timestamp());
745 render_times_ms_.push_back(input_frame.render_time_ms());
perkj9576e542015-11-12 06:43:16 -0800746 current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
747
perkj30e91822015-11-20 01:31:25 -0800748 if (!DeliverPendingOutputs(jni)) {
perkj9576e542015-11-12 06:43:16 -0800749 ALOGE << "Failed deliver pending outputs.";
750 ResetCodecOnCodecThread();
751 return WEBRTC_VIDEO_CODEC_ERROR;
752 }
753 return WEBRTC_VIDEO_CODEC_OK;
754}
755
756bool MediaCodecVideoEncoder::MaybeReconfigureEncoderOnCodecThread(
757 const webrtc::VideoFrame& frame) {
758 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
759
perkj30e91822015-11-20 01:31:25 -0800760 const bool is_texture_frame = frame.native_handle() != nullptr;
761 const bool reconfigure_due_to_format = is_texture_frame != use_surface_;
perkj9576e542015-11-12 06:43:16 -0800762 const bool reconfigure_due_to_size =
763 frame.width() != width_ || frame.height() != height_;
764
perkj30e91822015-11-20 01:31:25 -0800765 if (reconfigure_due_to_format) {
766 ALOGD << "Reconfigure encoder due to format change. "
767 << (use_surface_ ?
768 "Reconfiguring to encode from byte buffer." :
769 "Reconfiguring to encode from texture.");
glaznev94291482016-02-01 13:17:18 -0800770 LogStatistics(true);
perkj30e91822015-11-20 01:31:25 -0800771 }
perkj9576e542015-11-12 06:43:16 -0800772 if (reconfigure_due_to_size) {
glaznev94291482016-02-01 13:17:18 -0800773 ALOGW << "Reconfigure encoder due to frame resolution change from "
perkj9576e542015-11-12 06:43:16 -0800774 << width_ << " x " << height_ << " to " << frame.width() << " x "
775 << frame.height();
glaznev94291482016-02-01 13:17:18 -0800776 LogStatistics(true);
perkj9576e542015-11-12 06:43:16 -0800777 width_ = frame.width();
778 height_ = frame.height();
779 }
780
perkj30e91822015-11-20 01:31:25 -0800781 if (!reconfigure_due_to_format && !reconfigure_due_to_size)
perkj9576e542015-11-12 06:43:16 -0800782 return true;
783
784 ReleaseOnCodecThread();
785
perkj30e91822015-11-20 01:31:25 -0800786 return InitEncodeOnCodecThread(width_, height_, 0, 0 , is_texture_frame) ==
perkj9576e542015-11-12 06:43:16 -0800787 WEBRTC_VIDEO_CODEC_OK;
788}
789
790bool MediaCodecVideoEncoder::EncodeByteBufferOnCodecThread(JNIEnv* jni,
791 bool key_frame, const webrtc::VideoFrame& frame, int input_buffer_index) {
792 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
perkj30e91822015-11-20 01:31:25 -0800793 RTC_CHECK(!use_surface_);
perkj9576e542015-11-12 06:43:16 -0800794
perkj9576e542015-11-12 06:43:16 -0800795 jobject j_input_buffer = input_buffers_[input_buffer_index];
796 uint8_t* yuv_buffer =
797 reinterpret_cast<uint8_t*>(jni->GetDirectBufferAddress(j_input_buffer));
798 CHECK_EXCEPTION(jni);
799 RTC_CHECK(yuv_buffer) << "Indirect buffer??";
800 RTC_CHECK(!libyuv::ConvertFromI420(
801 frame.buffer(webrtc::kYPlane), frame.stride(webrtc::kYPlane),
802 frame.buffer(webrtc::kUPlane), frame.stride(webrtc::kUPlane),
803 frame.buffer(webrtc::kVPlane), frame.stride(webrtc::kVPlane),
804 yuv_buffer, width_, width_, height_, encoder_fourcc_))
805 << "ConvertFromI420 failed";
806
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000807 bool encode_status = jni->CallBooleanMethod(*j_media_codec_video_encoder_,
perkj9576e542015-11-12 06:43:16 -0800808 j_encode_buffer_method_,
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000809 key_frame,
perkj9576e542015-11-12 06:43:16 -0800810 input_buffer_index,
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000811 yuv_size_,
812 current_timestamp_us_);
813 CHECK_EXCEPTION(jni);
perkj9576e542015-11-12 06:43:16 -0800814 return encode_status;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000815}
816
perkj30e91822015-11-20 01:31:25 -0800817bool MediaCodecVideoEncoder::EncodeTextureOnCodecThread(JNIEnv* jni,
818 bool key_frame, const webrtc::VideoFrame& frame) {
819 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
820 RTC_CHECK(use_surface_);
821 NativeHandleImpl* handle =
822 static_cast<NativeHandleImpl*>(frame.native_handle());
823 jfloatArray sampling_matrix = jni->NewFloatArray(16);
824 jni->SetFloatArrayRegion(sampling_matrix, 0, 16, handle->sampling_matrix);
825
826 bool encode_status = jni->CallBooleanMethod(*j_media_codec_video_encoder_,
827 j_encode_texture_method_,
828 key_frame,
829 handle->oes_texture_id,
830 sampling_matrix,
831 current_timestamp_us_);
832 CHECK_EXCEPTION(jni);
833 return encode_status;
834}
835
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000836int32_t MediaCodecVideoEncoder::RegisterEncodeCompleteCallbackOnCodecThread(
837 webrtc::EncodedImageCallback* callback) {
perkj9576e542015-11-12 06:43:16 -0800838 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000839 JNIEnv* jni = AttachCurrentThreadIfNeeded();
840 ScopedLocalRefFrame local_ref_frame(jni);
841 callback_ = callback;
842 return WEBRTC_VIDEO_CODEC_OK;
843}
844
845int32_t MediaCodecVideoEncoder::ReleaseOnCodecThread() {
perkj9576e542015-11-12 06:43:16 -0800846 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000847 if (!inited_) {
848 return WEBRTC_VIDEO_CODEC_OK;
849 }
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000850 JNIEnv* jni = AttachCurrentThreadIfNeeded();
Alex Glaznevfddf6e52015-10-07 16:51:02 -0700851 ALOGD << "EncoderReleaseOnCodecThread: Frames received: " <<
852 frames_received_ << ". Encoded: " << frames_encoded_ <<
glaznev919ff752016-01-27 15:01:03 -0800853 ". Dropped: " << frames_dropped_media_encoder_;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000854 ScopedLocalRefFrame local_ref_frame(jni);
855 for (size_t i = 0; i < input_buffers_.size(); ++i)
856 jni->DeleteGlobalRef(input_buffers_[i]);
857 input_buffers_.clear();
858 jni->CallVoidMethod(*j_media_codec_video_encoder_, j_release_method_);
859 CHECK_EXCEPTION(jni);
860 rtc::MessageQueueManager::Clear(this);
861 inited_ = false;
perkj30e91822015-11-20 01:31:25 -0800862 use_surface_ = false;
Alex Glaznevfddf6e52015-10-07 16:51:02 -0700863 ALOGD << "EncoderReleaseOnCodecThread done.";
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000864 return WEBRTC_VIDEO_CODEC_OK;
865}
866
867int32_t MediaCodecVideoEncoder::SetRatesOnCodecThread(uint32_t new_bit_rate,
868 uint32_t frame_rate) {
perkj9576e542015-11-12 06:43:16 -0800869 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
glaznevf4decb52016-01-15 13:49:22 -0800870 frame_rate = (frame_rate < MAX_ALLOWED_VIDEO_FPS) ?
871 frame_rate : MAX_ALLOWED_VIDEO_FPS;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000872 if (last_set_bitrate_kbps_ == new_bit_rate &&
873 last_set_fps_ == frame_rate) {
874 return WEBRTC_VIDEO_CODEC_OK;
875 }
glaznev919ff752016-01-27 15:01:03 -0800876 if (scale_) {
877 quality_scaler_.ReportFramerate(frame_rate);
878 }
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000879 JNIEnv* jni = AttachCurrentThreadIfNeeded();
880 ScopedLocalRefFrame local_ref_frame(jni);
881 if (new_bit_rate > 0) {
882 last_set_bitrate_kbps_ = new_bit_rate;
883 }
884 if (frame_rate > 0) {
885 last_set_fps_ = frame_rate;
886 }
887 bool ret = jni->CallBooleanMethod(*j_media_codec_video_encoder_,
888 j_set_rates_method_,
889 last_set_bitrate_kbps_,
890 last_set_fps_);
891 CHECK_EXCEPTION(jni);
892 if (!ret) {
perkj9576e542015-11-12 06:43:16 -0800893 ResetCodecOnCodecThread();
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000894 return WEBRTC_VIDEO_CODEC_ERROR;
895 }
896 return WEBRTC_VIDEO_CODEC_OK;
897}
898
899int MediaCodecVideoEncoder::GetOutputBufferInfoIndex(
900 JNIEnv* jni,
901 jobject j_output_buffer_info) {
902 return GetIntField(jni, j_output_buffer_info, j_info_index_field_);
903}
904
905jobject MediaCodecVideoEncoder::GetOutputBufferInfoBuffer(
906 JNIEnv* jni,
907 jobject j_output_buffer_info) {
908 return GetObjectField(jni, j_output_buffer_info, j_info_buffer_field_);
909}
910
911bool MediaCodecVideoEncoder::GetOutputBufferInfoIsKeyFrame(
912 JNIEnv* jni,
913 jobject j_output_buffer_info) {
914 return GetBooleanField(jni, j_output_buffer_info, j_info_is_key_frame_field_);
915}
916
917jlong MediaCodecVideoEncoder::GetOutputBufferInfoPresentationTimestampUs(
918 JNIEnv* jni,
919 jobject j_output_buffer_info) {
920 return GetLongField(
921 jni, j_output_buffer_info, j_info_presentation_timestamp_us_field_);
922}
923
924bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) {
perkj9576e542015-11-12 06:43:16 -0800925 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000926 while (true) {
927 jobject j_output_buffer_info = jni->CallObjectMethod(
928 *j_media_codec_video_encoder_, j_dequeue_output_buffer_method_);
929 CHECK_EXCEPTION(jni);
930 if (IsNull(jni, j_output_buffer_info)) {
931 break;
932 }
933
934 int output_buffer_index =
935 GetOutputBufferInfoIndex(jni, j_output_buffer_info);
936 if (output_buffer_index == -1) {
perkj9576e542015-11-12 06:43:16 -0800937 ResetCodecOnCodecThread();
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000938 return false;
939 }
940
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000941 // Get key and config frame flags.
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000942 jobject j_output_buffer =
943 GetOutputBufferInfoBuffer(jni, j_output_buffer_info);
944 bool key_frame = GetOutputBufferInfoIsKeyFrame(jni, j_output_buffer_info);
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000945
946 // Get frame timestamps from a queue - for non config frames only.
947 int64_t frame_encoding_time_ms = 0;
948 last_output_timestamp_ms_ =
949 GetOutputBufferInfoPresentationTimestampUs(jni, j_output_buffer_info) /
950 1000;
951 if (frames_in_queue_ > 0) {
952 output_timestamp_ = timestamps_.front();
953 timestamps_.erase(timestamps_.begin());
954 output_render_time_ms_ = render_times_ms_.front();
955 render_times_ms_.erase(render_times_ms_.begin());
956 frame_encoding_time_ms = GetCurrentTimeMs() - frame_rtc_times_ms_.front();
957 frame_rtc_times_ms_.erase(frame_rtc_times_ms_.begin());
958 frames_in_queue_--;
959 }
960
961 // Extract payload.
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000962 size_t payload_size = jni->GetDirectBufferCapacity(j_output_buffer);
Peter Boström0c4e06b2015-10-07 12:23:21 +0200963 uint8_t* payload = reinterpret_cast<uint8_t*>(
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000964 jni->GetDirectBufferAddress(j_output_buffer));
965 CHECK_EXCEPTION(jni);
966
glaznevf4decb52016-01-15 13:49:22 -0800967 if (frames_encoded_ < kMaxEncodedLogFrames) {
glaznev94291482016-02-01 13:17:18 -0800968 int current_latency =
969 (int)(last_input_timestamp_ms_ - last_output_timestamp_ms_);
970 ALOGD << "Encoder frame out # " << frames_encoded_ <<
971 ". Key: " << key_frame <<
972 ". Size: " << payload_size <<
973 ". TS: " << (int)last_output_timestamp_ms_ <<
974 ". Latency: " << current_latency <<
glaznevf4decb52016-01-15 13:49:22 -0800975 ". EncTime: " << frame_encoding_time_ms;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000976 }
977
978 // Callback - return encoded frame.
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000979 int32_t callback_status = 0;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000980 if (callback_) {
981 scoped_ptr<webrtc::EncodedImage> image(
982 new webrtc::EncodedImage(payload, payload_size, payload_size));
983 image->_encodedWidth = width_;
984 image->_encodedHeight = height_;
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000985 image->_timeStamp = output_timestamp_;
986 image->capture_time_ms_ = output_render_time_ms_;
Peter Boström49e196a2015-10-23 15:58:18 +0200987 image->_frameType =
988 (key_frame ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta);
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000989 image->_completeFrame = true;
asapersson075fb4b2015-10-29 08:49:14 -0700990 image->adapt_reason_.quality_resolution_downscales =
991 scale_ ? quality_scaler_.downscale_shift() : -1;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000992
993 webrtc::CodecSpecificInfo info;
994 memset(&info, 0, sizeof(info));
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000995 info.codecType = codecType_;
996 if (codecType_ == kVideoCodecVP8) {
997 info.codecSpecific.VP8.pictureId = picture_id_;
998 info.codecSpecific.VP8.nonReference = false;
999 info.codecSpecific.VP8.simulcastIdx = 0;
1000 info.codecSpecific.VP8.temporalIdx = webrtc::kNoTemporalIdx;
1001 info.codecSpecific.VP8.layerSync = false;
1002 info.codecSpecific.VP8.tl0PicIdx = webrtc::kNoTl0PicIdx;
1003 info.codecSpecific.VP8.keyIdx = webrtc::kNoKeyIdx;
Alex Glaznevad948c42015-11-18 13:06:42 -08001004 } else if (codecType_ == kVideoCodecVP9) {
1005 if (key_frame) {
1006 gof_idx_ = 0;
1007 }
1008 info.codecSpecific.VP9.picture_id = picture_id_;
1009 info.codecSpecific.VP9.inter_pic_predicted = key_frame ? false : true;
1010 info.codecSpecific.VP9.flexible_mode = false;
1011 info.codecSpecific.VP9.ss_data_available = key_frame ? true : false;
1012 info.codecSpecific.VP9.tl0_pic_idx = tl0_pic_idx_++;
1013 info.codecSpecific.VP9.temporal_idx = webrtc::kNoTemporalIdx;
1014 info.codecSpecific.VP9.spatial_idx = webrtc::kNoSpatialIdx;
1015 info.codecSpecific.VP9.temporal_up_switch = true;
1016 info.codecSpecific.VP9.inter_layer_predicted = false;
1017 info.codecSpecific.VP9.gof_idx =
1018 static_cast<uint8_t>(gof_idx_++ % gof_.num_frames_in_gof);
1019 info.codecSpecific.VP9.num_spatial_layers = 1;
1020 info.codecSpecific.VP9.spatial_layer_resolution_present = false;
1021 if (info.codecSpecific.VP9.ss_data_available) {
1022 info.codecSpecific.VP9.spatial_layer_resolution_present = true;
1023 info.codecSpecific.VP9.width[0] = width_;
1024 info.codecSpecific.VP9.height[0] = height_;
1025 info.codecSpecific.VP9.gof.CopyGofInfoVP9(gof_);
1026 }
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001027 }
Alex Glaznevad948c42015-11-18 13:06:42 -08001028 picture_id_ = (picture_id_ + 1) & 0x7FFF;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001029
1030 // Generate a header describing a single fragment.
1031 webrtc::RTPFragmentationHeader header;
1032 memset(&header, 0, sizeof(header));
Alex Glaznevad948c42015-11-18 13:06:42 -08001033 if (codecType_ == kVideoCodecVP8 || codecType_ == kVideoCodecVP9) {
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001034 header.VerifyAndAllocateFragmentationHeader(1);
1035 header.fragmentationOffset[0] = 0;
1036 header.fragmentationLength[0] = image->_length;
1037 header.fragmentationPlType[0] = 0;
1038 header.fragmentationTimeDiff[0] = 0;
Alex Glaznevad948c42015-11-18 13:06:42 -08001039 if (codecType_ == kVideoCodecVP8 && scale_) {
asapersson86b01602015-10-20 23:55:26 -07001040 int qp;
glaznevf4decb52016-01-15 13:49:22 -08001041 if (webrtc::vp8::GetQp(payload, payload_size, &qp)) {
1042 current_acc_qp_ += qp;
asapersson86b01602015-10-20 23:55:26 -07001043 quality_scaler_.ReportQP(qp);
glaznevf4decb52016-01-15 13:49:22 -08001044 }
asapersson86b01602015-10-20 23:55:26 -07001045 }
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001046 } else if (codecType_ == kVideoCodecH264) {
Peter Boström2bc68c72015-09-24 16:22:28 +02001047 if (scale_) {
1048 h264_bitstream_parser_.ParseBitstream(payload, payload_size);
1049 int qp;
glaznevf4decb52016-01-15 13:49:22 -08001050 if (h264_bitstream_parser_.GetLastSliceQp(&qp)) {
1051 current_acc_qp_ += qp;
Peter Boström2bc68c72015-09-24 16:22:28 +02001052 quality_scaler_.ReportQP(qp);
glaznevf4decb52016-01-15 13:49:22 -08001053 }
Peter Boström2bc68c72015-09-24 16:22:28 +02001054 }
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001055 // For H.264 search for start codes.
1056 int32_t scPositions[MAX_NALUS_PERFRAME + 1] = {};
1057 int32_t scPositionsLength = 0;
1058 int32_t scPosition = 0;
1059 while (scPositionsLength < MAX_NALUS_PERFRAME) {
1060 int32_t naluPosition = NextNaluPosition(
1061 payload + scPosition, payload_size - scPosition);
1062 if (naluPosition < 0) {
1063 break;
1064 }
1065 scPosition += naluPosition;
1066 scPositions[scPositionsLength++] = scPosition;
1067 scPosition += H264_SC_LENGTH;
1068 }
1069 if (scPositionsLength == 0) {
Alex Glaznevfddf6e52015-10-07 16:51:02 -07001070 ALOGE << "Start code is not found!";
1071 ALOGE << "Data:" << image->_buffer[0] << " " << image->_buffer[1]
1072 << " " << image->_buffer[2] << " " << image->_buffer[3]
1073 << " " << image->_buffer[4] << " " << image->_buffer[5];
perkj9576e542015-11-12 06:43:16 -08001074 ResetCodecOnCodecThread();
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001075 return false;
1076 }
1077 scPositions[scPositionsLength] = payload_size;
1078 header.VerifyAndAllocateFragmentationHeader(scPositionsLength);
1079 for (size_t i = 0; i < scPositionsLength; i++) {
1080 header.fragmentationOffset[i] = scPositions[i] + H264_SC_LENGTH;
1081 header.fragmentationLength[i] =
1082 scPositions[i + 1] - header.fragmentationOffset[i];
1083 header.fragmentationPlType[i] = 0;
1084 header.fragmentationTimeDiff[i] = 0;
1085 }
1086 }
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001087
1088 callback_status = callback_->Encoded(*image, &info, &header);
1089 }
1090
1091 // Return output buffer back to the encoder.
1092 bool success = jni->CallBooleanMethod(*j_media_codec_video_encoder_,
1093 j_release_output_buffer_method_,
1094 output_buffer_index);
1095 CHECK_EXCEPTION(jni);
1096 if (!success) {
perkj9576e542015-11-12 06:43:16 -08001097 ResetCodecOnCodecThread();
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001098 return false;
1099 }
1100
glaznevf4decb52016-01-15 13:49:22 -08001101 // Calculate and print encoding statistics - every 3 seconds.
1102 frames_encoded_++;
1103 current_frames_++;
1104 current_bytes_ += payload_size;
1105 current_encoding_time_ms_ += frame_encoding_time_ms;
glaznev94291482016-02-01 13:17:18 -08001106 LogStatistics(false);
glaznevf4decb52016-01-15 13:49:22 -08001107
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001108 if (callback_status > 0) {
1109 drop_next_input_frame_ = true;
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001110 // Theoretically could handle callback_status<0 here, but unclear what
1111 // that would mean for us.
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001112 }
1113 }
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001114 return true;
1115}
1116
glaznev94291482016-02-01 13:17:18 -08001117void MediaCodecVideoEncoder::LogStatistics(bool force_log) {
1118 int statistic_time_ms = GetCurrentTimeMs() - stat_start_time_ms_;
1119 if ((statistic_time_ms >= kMediaCodecStatisticsIntervalMs || force_log) &&
1120 current_frames_ > 0 && statistic_time_ms > 0) {
1121 int current_bitrate = current_bytes_ * 8 / statistic_time_ms;
1122 int current_fps =
1123 (current_frames_ * 1000 + statistic_time_ms / 2) / statistic_time_ms;
1124 ALOGD << "Encoded frames: " << frames_encoded_ <<
1125 ". Bitrate: " << current_bitrate <<
1126 ", target: " << last_set_bitrate_kbps_ << " kbps" <<
1127 ", fps: " << current_fps <<
1128 ", encTime: " << (current_encoding_time_ms_ / current_frames_) <<
1129 ". QP: " << (current_acc_qp_ / current_frames_) <<
1130 " for last " << statistic_time_ms << " ms.";
1131 stat_start_time_ms_ = GetCurrentTimeMs();
1132 current_frames_ = 0;
1133 current_bytes_ = 0;
1134 current_acc_qp_ = 0;
1135 current_encoding_time_ms_ = 0;
1136 }
1137}
1138
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001139int32_t MediaCodecVideoEncoder::NextNaluPosition(
1140 uint8_t *buffer, size_t buffer_size) {
1141 if (buffer_size < H264_SC_LENGTH) {
1142 return -1;
1143 }
1144 uint8_t *head = buffer;
1145 // Set end buffer pointer to 4 bytes before actual buffer end so we can
1146 // access head[1], head[2] and head[3] in a loop without buffer overrun.
1147 uint8_t *end = buffer + buffer_size - H264_SC_LENGTH;
1148
1149 while (head < end) {
1150 if (head[0]) {
1151 head++;
1152 continue;
1153 }
1154 if (head[1]) { // got 00xx
1155 head += 2;
1156 continue;
1157 }
1158 if (head[2]) { // got 0000xx
1159 head += 3;
1160 continue;
1161 }
1162 if (head[3] != 0x01) { // got 000000xx
glaznev@webrtc.orgdc08a232015-03-06 23:32:20 +00001163 head++; // xx != 1, continue searching.
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001164 continue;
1165 }
1166 return (int32_t)(head - buffer);
1167 }
1168 return -1;
1169}
1170
jackychen61b4d512015-04-21 15:30:11 -07001171void MediaCodecVideoEncoder::OnDroppedFrame() {
glaznevf4decb52016-01-15 13:49:22 -08001172 // Report dropped frame to quality_scaler_.
Peter Boström2bc68c72015-09-24 16:22:28 +02001173 if (scale_)
1174 quality_scaler_.ReportDroppedFrame();
jackychen61b4d512015-04-21 15:30:11 -07001175}
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001176
jackychen6e2ce6e2015-07-13 16:26:33 -07001177int MediaCodecVideoEncoder::GetTargetFramerate() {
Peter Boström2bc68c72015-09-24 16:22:28 +02001178 return scale_ ? quality_scaler_.GetTargetFramerate() : -1;
jackychen6e2ce6e2015-07-13 16:26:33 -07001179}
1180
Peter Boströmb7d9a972015-12-18 16:01:11 +01001181const char* MediaCodecVideoEncoder::ImplementationName() const {
1182 return "MediaCodec";
1183}
1184
Perec2922f2016-01-27 15:25:46 +01001185MediaCodecVideoEncoderFactory::MediaCodecVideoEncoderFactory() {
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001186 JNIEnv* jni = AttachCurrentThreadIfNeeded();
1187 ScopedLocalRefFrame local_ref_frame(jni);
1188 jclass j_encoder_class = FindClass(jni, "org/webrtc/MediaCodecVideoEncoder");
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001189 supported_codecs_.clear();
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001190
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001191 bool is_vp8_hw_supported = jni->CallStaticBooleanMethod(
1192 j_encoder_class,
1193 GetStaticMethodID(jni, j_encoder_class, "isVp8HwSupported", "()Z"));
1194 CHECK_EXCEPTION(jni);
1195 if (is_vp8_hw_supported) {
Alex Glaznevfddf6e52015-10-07 16:51:02 -07001196 ALOGD << "VP8 HW Encoder supported.";
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001197 supported_codecs_.push_back(VideoCodec(kVideoCodecVP8, "VP8",
1198 MAX_VIDEO_WIDTH, MAX_VIDEO_HEIGHT, MAX_VIDEO_FPS));
1199 }
1200
Alex Glaznevad948c42015-11-18 13:06:42 -08001201 bool is_vp9_hw_supported = jni->CallStaticBooleanMethod(
1202 j_encoder_class,
1203 GetStaticMethodID(jni, j_encoder_class, "isVp9HwSupported", "()Z"));
1204 CHECK_EXCEPTION(jni);
1205 if (is_vp9_hw_supported) {
1206 ALOGD << "VP9 HW Encoder supported.";
1207 supported_codecs_.push_back(VideoCodec(kVideoCodecVP9, "VP9",
1208 MAX_VIDEO_WIDTH, MAX_VIDEO_HEIGHT, MAX_VIDEO_FPS));
1209 }
1210
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001211 bool is_h264_hw_supported = jni->CallStaticBooleanMethod(
1212 j_encoder_class,
1213 GetStaticMethodID(jni, j_encoder_class, "isH264HwSupported", "()Z"));
1214 CHECK_EXCEPTION(jni);
1215 if (is_h264_hw_supported) {
Alex Glaznevfddf6e52015-10-07 16:51:02 -07001216 ALOGD << "H.264 HW Encoder supported.";
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001217 supported_codecs_.push_back(VideoCodec(kVideoCodecH264, "H264",
1218 MAX_VIDEO_WIDTH, MAX_VIDEO_HEIGHT, MAX_VIDEO_FPS));
1219 }
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001220}
1221
Perec2922f2016-01-27 15:25:46 +01001222MediaCodecVideoEncoderFactory::~MediaCodecVideoEncoderFactory() {
1223 ALOGD << "MediaCodecVideoEncoderFactory dtor";
1224}
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001225
perkj30e91822015-11-20 01:31:25 -08001226void MediaCodecVideoEncoderFactory::SetEGLContext(
1227 JNIEnv* jni, jobject render_egl_context) {
1228 ALOGD << "MediaCodecVideoEncoderFactory::SetEGLContext";
Perec2922f2016-01-27 15:25:46 +01001229 if (!egl_base_.CreateEglBase(jni, render_egl_context)) {
1230 ALOGW << "Invalid EGL context - HW surface encoding is disabled.";
perkj30e91822015-11-20 01:31:25 -08001231 }
1232}
1233
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001234webrtc::VideoEncoder* MediaCodecVideoEncoderFactory::CreateVideoEncoder(
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001235 VideoCodecType type) {
1236 if (supported_codecs_.empty()) {
Alex Glaznevad948c42015-11-18 13:06:42 -08001237 ALOGW << "No HW video encoder for type " << (int)type;
Perec2922f2016-01-27 15:25:46 +01001238 return nullptr;
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001239 }
1240 for (std::vector<VideoCodec>::const_iterator it = supported_codecs_.begin();
1241 it != supported_codecs_.end(); ++it) {
1242 if (it->type == type) {
Alex Glaznevfddf6e52015-10-07 16:51:02 -07001243 ALOGD << "Create HW video encoder for type " << (int)type <<
1244 " (" << it->name << ").";
perkj30e91822015-11-20 01:31:25 -08001245 return new MediaCodecVideoEncoder(AttachCurrentThreadIfNeeded(), type,
Perec2922f2016-01-27 15:25:46 +01001246 egl_base_.egl_base_context());
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001247 }
1248 }
Alex Glaznevad948c42015-11-18 13:06:42 -08001249 ALOGW << "Can not find HW video encoder for type " << (int)type;
Perec2922f2016-01-27 15:25:46 +01001250 return nullptr;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001251}
1252
1253const std::vector<MediaCodecVideoEncoderFactory::VideoCodec>&
1254MediaCodecVideoEncoderFactory::codecs() const {
1255 return supported_codecs_;
1256}
1257
1258void MediaCodecVideoEncoderFactory::DestroyVideoEncoder(
1259 webrtc::VideoEncoder* encoder) {
Alex Glaznevfddf6e52015-10-07 16:51:02 -07001260 ALOGD << "Destroy video encoder.";
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001261 delete encoder;
1262}
1263
1264} // namespace webrtc_jni
1265