blob: a9aa1d9531253185cb84f511c91b73f49271537d [file] [log] [blame]
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001/*
kjellanderb24317b2016-02-10 07:54:43 -08002 * Copyright 2015 The WebRTC project authors. All Rights Reserved.
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00003 *
kjellanderb24317b2016-02-10 07:54:43 -08004 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00009 */
10
kjellandera96e2d72016-02-04 23:52:28 -080011// NOTICE: androidmediaencoder_jni.h must be included before
12// androidmediacodeccommon.h to avoid build errors.
Henrik Kjellander15583c12016-02-10 10:53:12 +010013#include "webrtc/api/java/jni/androidmediaencoder_jni.h"
14
Perba7dc722016-04-19 15:01:23 +020015#include <algorithm>
16#include <list>
17
kjellandera96e2d72016-02-04 23:52:28 -080018#include "third_party/libyuv/include/libyuv/convert.h"
19#include "third_party/libyuv/include/libyuv/convert_from.h"
20#include "third_party/libyuv/include/libyuv/video_common.h"
Henrik Kjellander15583c12016-02-10 10:53:12 +010021#include "webrtc/api/java/jni/androidmediacodeccommon.h"
22#include "webrtc/api/java/jni/classreferenceholder.h"
23#include "webrtc/api/java/jni/native_handle_impl.h"
glaznev@webrtc.org18c92472015-02-18 18:42:55 +000024#include "webrtc/base/bind.h"
25#include "webrtc/base/checks.h"
26#include "webrtc/base/logging.h"
27#include "webrtc/base/thread.h"
perkj9576e542015-11-12 06:43:16 -080028#include "webrtc/base/thread_checker.h"
asapersson1d61a512016-01-20 01:13:46 -080029#include "webrtc/common_types.h"
Peter Boström2bc68c72015-09-24 16:22:28 +020030#include "webrtc/modules/rtp_rtcp/source/h264_bitstream_parser.h"
perkj30e91822015-11-20 01:31:25 -080031#include "webrtc/modules/video_coding/include/video_codec_interface.h"
kjellander@webrtc.orgb7ce9642015-11-18 23:04:10 +010032#include "webrtc/modules/video_coding/utility/quality_scaler.h"
33#include "webrtc/modules/video_coding/utility/vp8_header_parser.h"
Henrik Kjellander98f53512015-10-28 18:17:40 +010034#include "webrtc/system_wrappers/include/field_trial.h"
35#include "webrtc/system_wrappers/include/logcat_trace_context.h"
glaznev@webrtc.org18c92472015-02-18 18:42:55 +000036
37using rtc::Bind;
38using rtc::Thread;
39using rtc::ThreadManager;
40using rtc::scoped_ptr;
41
42using webrtc::CodecSpecificInfo;
43using webrtc::EncodedImage;
Miguel Casas-Sanchez47650702015-05-29 17:21:40 -070044using webrtc::VideoFrame;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +000045using webrtc::RTPFragmentationHeader;
46using webrtc::VideoCodec;
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +000047using webrtc::VideoCodecType;
48using webrtc::kVideoCodecH264;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +000049using webrtc::kVideoCodecVP8;
Alex Glaznevad948c42015-11-18 13:06:42 -080050using webrtc::kVideoCodecVP9;
Alex Glazneva9d08922016-02-19 15:24:06 -080051using webrtc::QualityScaler;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +000052
53namespace webrtc_jni {
54
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +000055// H.264 start code length.
56#define H264_SC_LENGTH 4
57// Maximum allowed NALUs in one output frame.
58#define MAX_NALUS_PERFRAME 32
59// Maximum supported HW video encoder resolution.
60#define MAX_VIDEO_WIDTH 1280
61#define MAX_VIDEO_HEIGHT 1280
62// Maximum supported HW video encoder fps.
63#define MAX_VIDEO_FPS 30
glaznevf4decb52016-01-15 13:49:22 -080064// Maximum allowed fps value in SetRates() call.
65#define MAX_ALLOWED_VIDEO_FPS 60
66// Maximum allowed frames in encoder input queue.
67#define MAX_ENCODER_Q_SIZE 2
glaznev919ff752016-01-27 15:01:03 -080068// Maximum amount of dropped frames caused by full encoder queue - exceeding
69// this threshold means that encoder probably got stuck and need to be reset.
70#define ENCODER_STALL_FRAMEDROP_THRESHOLD 60
glaznevf4decb52016-01-15 13:49:22 -080071
72// Logging macros.
73#define TAG_ENCODER "MediaCodecVideoEncoder"
74#ifdef TRACK_BUFFER_TIMING
75#define ALOGV(...)
76 __android_log_print(ANDROID_LOG_VERBOSE, TAG_ENCODER, __VA_ARGS__)
77#else
78#define ALOGV(...)
79#endif
80#define ALOGD LOG_TAG(rtc::LS_INFO, TAG_ENCODER)
81#define ALOGW LOG_TAG(rtc::LS_WARNING, TAG_ENCODER)
82#define ALOGE LOG_TAG(rtc::LS_ERROR, TAG_ENCODER)
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +000083
asapersson1d61a512016-01-20 01:13:46 -080084namespace {
85// Maximum time limit between incoming frames before requesting a key frame.
86const size_t kFrameDiffThresholdMs = 1100;
87const int kMinKeyFrameInterval = 2;
88} // namespace
89
glaznev@webrtc.org18c92472015-02-18 18:42:55 +000090// MediaCodecVideoEncoder is a webrtc::VideoEncoder implementation that uses
91// Android's MediaCodec SDK API behind the scenes to implement (hopefully)
92// HW-backed video encode. This C++ class is implemented as a very thin shim,
93// delegating all of the interesting work to org.webrtc.MediaCodecVideoEncoder.
94// MediaCodecVideoEncoder is created, operated, and destroyed on a single
95// thread, currently the libjingle Worker thread.
96class MediaCodecVideoEncoder : public webrtc::VideoEncoder,
97 public rtc::MessageHandler {
98 public:
99 virtual ~MediaCodecVideoEncoder();
perkj9576e542015-11-12 06:43:16 -0800100 MediaCodecVideoEncoder(JNIEnv* jni,
perkj30e91822015-11-20 01:31:25 -0800101 VideoCodecType codecType,
102 jobject egl_context);
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000103
104 // webrtc::VideoEncoder implementation. Everything trampolines to
105 // |codec_thread_| for execution.
106 int32_t InitEncode(const webrtc::VideoCodec* codec_settings,
107 int32_t /* number_of_cores */,
108 size_t /* max_payload_size */) override;
pbos22993e12015-10-19 02:39:06 -0700109 int32_t Encode(const webrtc::VideoFrame& input_image,
110 const webrtc::CodecSpecificInfo* /* codec_specific_info */,
111 const std::vector<webrtc::FrameType>* frame_types) override;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000112 int32_t RegisterEncodeCompleteCallback(
113 webrtc::EncodedImageCallback* callback) override;
114 int32_t Release() override;
115 int32_t SetChannelParameters(uint32_t /* packet_loss */,
116 int64_t /* rtt */) override;
117 int32_t SetRates(uint32_t new_bit_rate, uint32_t frame_rate) override;
118
119 // rtc::MessageHandler implementation.
120 void OnMessage(rtc::Message* msg) override;
121
jackychen61b4d512015-04-21 15:30:11 -0700122 void OnDroppedFrame() override;
123
Perec2922f2016-01-27 15:25:46 +0100124 bool SupportsNativeHandle() const override { return egl_context_ != nullptr; }
Peter Boströmb7d9a972015-12-18 16:01:11 +0100125 const char* ImplementationName() const override;
126
127 private:
128 // CHECK-fail if not running on |codec_thread_|.
129 void CheckOnCodecThread();
perkj30e91822015-11-20 01:31:25 -0800130
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000131 private:
perkj9576e542015-11-12 06:43:16 -0800132 // ResetCodecOnCodecThread() calls ReleaseOnCodecThread() and
133 // InitEncodeOnCodecThread() in an attempt to restore the codec to an
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000134 // operable state. Necessary after all manner of OMX-layer errors.
perkj9576e542015-11-12 06:43:16 -0800135 bool ResetCodecOnCodecThread();
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000136
137 // Implementation of webrtc::VideoEncoder methods above, all running on the
138 // codec thread exclusively.
139 //
140 // If width==0 then this is assumed to be a re-initialization and the
141 // previously-current values are reused instead of the passed parameters
142 // (makes it easier to reason about thread-safety).
perkj30e91822015-11-20 01:31:25 -0800143 int32_t InitEncodeOnCodecThread(int width, int height, int kbps, int fps,
144 bool use_surface);
145 // Reconfigure to match |frame| in width, height. Also reconfigures the
146 // encoder if |frame| is a texture/byte buffer and the encoder is initialized
147 // for byte buffer/texture. Returns false if reconfiguring fails.
perkj9576e542015-11-12 06:43:16 -0800148 bool MaybeReconfigureEncoderOnCodecThread(const webrtc::VideoFrame& frame);
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000149 int32_t EncodeOnCodecThread(
Miguel Casas-Sanchez47650702015-05-29 17:21:40 -0700150 const webrtc::VideoFrame& input_image,
pbos22993e12015-10-19 02:39:06 -0700151 const std::vector<webrtc::FrameType>* frame_types);
perkj9576e542015-11-12 06:43:16 -0800152 bool EncodeByteBufferOnCodecThread(JNIEnv* jni,
153 bool key_frame, const webrtc::VideoFrame& frame, int input_buffer_index);
perkj30e91822015-11-20 01:31:25 -0800154 bool EncodeTextureOnCodecThread(JNIEnv* jni,
155 bool key_frame, const webrtc::VideoFrame& frame);
perkj9576e542015-11-12 06:43:16 -0800156
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000157 int32_t RegisterEncodeCompleteCallbackOnCodecThread(
158 webrtc::EncodedImageCallback* callback);
159 int32_t ReleaseOnCodecThread();
160 int32_t SetRatesOnCodecThread(uint32_t new_bit_rate, uint32_t frame_rate);
161
162 // Helper accessors for MediaCodecVideoEncoder$OutputBufferInfo members.
163 int GetOutputBufferInfoIndex(JNIEnv* jni, jobject j_output_buffer_info);
164 jobject GetOutputBufferInfoBuffer(JNIEnv* jni, jobject j_output_buffer_info);
165 bool GetOutputBufferInfoIsKeyFrame(JNIEnv* jni, jobject j_output_buffer_info);
166 jlong GetOutputBufferInfoPresentationTimestampUs(
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000167 JNIEnv* jni, jobject j_output_buffer_info);
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000168
169 // Deliver any outputs pending in the MediaCodec to our |callback_| and return
170 // true on success.
171 bool DeliverPendingOutputs(JNIEnv* jni);
172
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000173 // Search for H.264 start codes.
174 int32_t NextNaluPosition(uint8_t *buffer, size_t buffer_size);
175
glaznev94291482016-02-01 13:17:18 -0800176 // Displays encoder statistics.
177 void LogStatistics(bool force_log);
178
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000179 // Type of video codec.
180 VideoCodecType codecType_;
181
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000182 // Valid all the time since RegisterEncodeCompleteCallback() Invoke()s to
183 // |codec_thread_| synchronously.
184 webrtc::EncodedImageCallback* callback_;
185
186 // State that is constant for the lifetime of this object once the ctor
187 // returns.
188 scoped_ptr<Thread> codec_thread_; // Thread on which to operate MediaCodec.
perkj9576e542015-11-12 06:43:16 -0800189 rtc::ThreadChecker codec_thread_checker_;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000190 ScopedGlobalRef<jclass> j_media_codec_video_encoder_class_;
191 ScopedGlobalRef<jobject> j_media_codec_video_encoder_;
192 jmethodID j_init_encode_method_;
perkj9576e542015-11-12 06:43:16 -0800193 jmethodID j_get_input_buffers_method_;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000194 jmethodID j_dequeue_input_buffer_method_;
perkj9576e542015-11-12 06:43:16 -0800195 jmethodID j_encode_buffer_method_;
perkj30e91822015-11-20 01:31:25 -0800196 jmethodID j_encode_texture_method_;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000197 jmethodID j_release_method_;
198 jmethodID j_set_rates_method_;
199 jmethodID j_dequeue_output_buffer_method_;
200 jmethodID j_release_output_buffer_method_;
201 jfieldID j_color_format_field_;
202 jfieldID j_info_index_field_;
203 jfieldID j_info_buffer_field_;
204 jfieldID j_info_is_key_frame_field_;
205 jfieldID j_info_presentation_timestamp_us_field_;
206
207 // State that is valid only between InitEncode() and the next Release().
208 // Touched only on codec_thread_ so no explicit synchronization necessary.
209 int width_; // Frame width in pixels.
210 int height_; // Frame height in pixels.
211 bool inited_;
perkj30e91822015-11-20 01:31:25 -0800212 bool use_surface_;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000213 uint16_t picture_id_;
214 enum libyuv::FourCC encoder_fourcc_; // Encoder color space format.
215 int last_set_bitrate_kbps_; // Last-requested bitrate in kbps.
216 int last_set_fps_; // Last-requested frame rate.
217 int64_t current_timestamp_us_; // Current frame timestamps in us.
218 int frames_received_; // Number of frames received by encoder.
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000219 int frames_encoded_; // Number of frames encoded by encoder.
glaznev919ff752016-01-27 15:01:03 -0800220 int frames_dropped_media_encoder_; // Number of frames dropped by encoder.
221 // Number of dropped frames caused by full queue.
222 int consecutive_full_queue_frame_drops_;
glaznev94291482016-02-01 13:17:18 -0800223 int64_t stat_start_time_ms_; // Start time for statistics.
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000224 int current_frames_; // Number of frames in the current statistics interval.
225 int current_bytes_; // Encoded bytes in the current statistics interval.
glaznevf4decb52016-01-15 13:49:22 -0800226 int current_acc_qp_; // Accumulated QP in the current statistics interval.
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000227 int current_encoding_time_ms_; // Overall encoding time in the current second
228 int64_t last_input_timestamp_ms_; // Timestamp of last received yuv frame.
229 int64_t last_output_timestamp_ms_; // Timestamp of last encoded frame.
Perba7dc722016-04-19 15:01:23 +0200230
231 struct InputFrameInfo {
232 InputFrameInfo(int64_t encode_start_time,
233 int32_t frame_timestamp,
234 int64_t frame_render_time_ms,
235 webrtc::VideoRotation rotation)
236 : encode_start_time(encode_start_time),
237 frame_timestamp(frame_timestamp),
238 frame_render_time_ms(frame_render_time_ms),
239 rotation(rotation) {}
240 // Time when video frame is sent to encoder input.
241 const int64_t encode_start_time;
242
243 // Input frame information.
244 const int32_t frame_timestamp;
245 const int64_t frame_render_time_ms;
246 const webrtc::VideoRotation rotation;
247 };
248 std::list<InputFrameInfo> input_frame_infos_;
249 int32_t output_timestamp_; // Last output frame timestamp from
250 // |input_frame_infos_|.
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000251 int64_t output_render_time_ms_; // Last output frame render time from
Perba7dc722016-04-19 15:01:23 +0200252 // |input_frame_infos_|.
253 webrtc::VideoRotation output_rotation_; // Last output frame rotation from
254 // |input_frame_infos_|.
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000255 // Frame size in bytes fed to MediaCodec.
256 int yuv_size_;
257 // True only when between a callback_->Encoded() call return a positive value
258 // and the next Encode() call being ignored.
259 bool drop_next_input_frame_;
260 // Global references; must be deleted in Release().
261 std::vector<jobject> input_buffers_;
Alex Glazneva9d08922016-02-19 15:24:06 -0800262 QualityScaler quality_scaler_;
jackychen61b4d512015-04-21 15:30:11 -0700263 // Dynamic resolution change, off by default.
264 bool scale_;
Peter Boström2bc68c72015-09-24 16:22:28 +0200265
266 // H264 bitstream parser, used to extract QP from encoded bitstreams.
267 webrtc::H264BitstreamParser h264_bitstream_parser_;
Alex Glaznevad948c42015-11-18 13:06:42 -0800268
269 // VP9 variables to populate codec specific structure.
270 webrtc::GofInfoVP9 gof_; // Contains each frame's temporal information for
271 // non-flexible VP9 mode.
272 uint8_t tl0_pic_idx_;
273 size_t gof_idx_;
perkj30e91822015-11-20 01:31:25 -0800274
275 // EGL context - owned by factory, should not be allocated/destroyed
276 // by MediaCodecVideoEncoder.
277 jobject egl_context_;
asapersson1d61a512016-01-20 01:13:46 -0800278
279 // Temporary fix for VP8.
280 // Sends a key frame if frames are largely spaced apart (possibly
281 // corresponding to a large image change).
282 int64_t last_frame_received_ms_;
283 int frames_received_since_last_key_;
284 webrtc::VideoCodecMode codec_mode_;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000285};
286
287MediaCodecVideoEncoder::~MediaCodecVideoEncoder() {
288 // Call Release() to ensure no more callbacks to us after we are deleted.
289 Release();
290}
291
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000292MediaCodecVideoEncoder::MediaCodecVideoEncoder(
perkj30e91822015-11-20 01:31:25 -0800293 JNIEnv* jni, VideoCodecType codecType, jobject egl_context) :
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000294 codecType_(codecType),
295 callback_(NULL),
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000296 codec_thread_(new Thread()),
297 j_media_codec_video_encoder_class_(
298 jni,
299 FindClass(jni, "org/webrtc/MediaCodecVideoEncoder")),
300 j_media_codec_video_encoder_(
301 jni,
302 jni->NewObject(*j_media_codec_video_encoder_class_,
303 GetMethodID(jni,
304 *j_media_codec_video_encoder_class_,
305 "<init>",
perkj30e91822015-11-20 01:31:25 -0800306 "()V"))),
kjellander60ca31b2016-01-04 10:15:53 -0800307 inited_(false),
308 use_surface_(false),
309 picture_id_(0),
perkj30e91822015-11-20 01:31:25 -0800310 egl_context_(egl_context) {
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000311 ScopedLocalRefFrame local_ref_frame(jni);
312 // It would be nice to avoid spinning up a new thread per MediaCodec, and
313 // instead re-use e.g. the PeerConnectionFactory's |worker_thread_|, but bug
314 // 2732 means that deadlocks abound. This class synchronously trampolines
315 // to |codec_thread_|, so if anything else can be coming to _us_ from
316 // |codec_thread_|, or from any thread holding the |_sendCritSect| described
317 // in the bug, we have a problem. For now work around that with a dedicated
318 // thread.
319 codec_thread_->SetName("MediaCodecVideoEncoder", NULL);
henrikg91d6ede2015-09-17 00:24:34 -0700320 RTC_CHECK(codec_thread_->Start()) << "Failed to start MediaCodecVideoEncoder";
perkj9576e542015-11-12 06:43:16 -0800321 codec_thread_checker_.DetachFromThread();
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000322 jclass j_output_buffer_info_class =
323 FindClass(jni, "org/webrtc/MediaCodecVideoEncoder$OutputBufferInfo");
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000324 j_init_encode_method_ = GetMethodID(
325 jni,
326 *j_media_codec_video_encoder_class_,
327 "initEncode",
perkj30e91822015-11-20 01:31:25 -0800328 "(Lorg/webrtc/MediaCodecVideoEncoder$VideoCodecType;"
perkj48477c12015-12-18 00:34:37 -0800329 "IIIILorg/webrtc/EglBase14$Context;)Z");
perkj9576e542015-11-12 06:43:16 -0800330 j_get_input_buffers_method_ = GetMethodID(
331 jni,
332 *j_media_codec_video_encoder_class_,
333 "getInputBuffers",
334 "()[Ljava/nio/ByteBuffer;");
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000335 j_dequeue_input_buffer_method_ = GetMethodID(
336 jni, *j_media_codec_video_encoder_class_, "dequeueInputBuffer", "()I");
perkj9576e542015-11-12 06:43:16 -0800337 j_encode_buffer_method_ = GetMethodID(
338 jni, *j_media_codec_video_encoder_class_, "encodeBuffer", "(ZIIJ)Z");
perkj30e91822015-11-20 01:31:25 -0800339 j_encode_texture_method_ = GetMethodID(
340 jni, *j_media_codec_video_encoder_class_, "encodeTexture",
341 "(ZI[FJ)Z");
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000342 j_release_method_ =
343 GetMethodID(jni, *j_media_codec_video_encoder_class_, "release", "()V");
344 j_set_rates_method_ = GetMethodID(
345 jni, *j_media_codec_video_encoder_class_, "setRates", "(II)Z");
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000346 j_dequeue_output_buffer_method_ = GetMethodID(
347 jni,
348 *j_media_codec_video_encoder_class_,
349 "dequeueOutputBuffer",
350 "()Lorg/webrtc/MediaCodecVideoEncoder$OutputBufferInfo;");
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000351 j_release_output_buffer_method_ = GetMethodID(
352 jni, *j_media_codec_video_encoder_class_, "releaseOutputBuffer", "(I)Z");
353
354 j_color_format_field_ =
355 GetFieldID(jni, *j_media_codec_video_encoder_class_, "colorFormat", "I");
356 j_info_index_field_ =
357 GetFieldID(jni, j_output_buffer_info_class, "index", "I");
358 j_info_buffer_field_ = GetFieldID(
359 jni, j_output_buffer_info_class, "buffer", "Ljava/nio/ByteBuffer;");
360 j_info_is_key_frame_field_ =
361 GetFieldID(jni, j_output_buffer_info_class, "isKeyFrame", "Z");
362 j_info_presentation_timestamp_us_field_ = GetFieldID(
363 jni, j_output_buffer_info_class, "presentationTimestampUs", "J");
364 CHECK_EXCEPTION(jni) << "MediaCodecVideoEncoder ctor failed";
Alex Glaznevad948c42015-11-18 13:06:42 -0800365 srand(time(NULL));
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000366 AllowBlockingCalls();
367}
368
369int32_t MediaCodecVideoEncoder::InitEncode(
370 const webrtc::VideoCodec* codec_settings,
371 int32_t /* number_of_cores */,
372 size_t /* max_payload_size */) {
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000373 if (codec_settings == NULL) {
Alex Glaznevfddf6e52015-10-07 16:51:02 -0700374 ALOGE << "NULL VideoCodec instance";
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000375 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
376 }
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000377 // Factory should guard against other codecs being used with us.
henrikg91d6ede2015-09-17 00:24:34 -0700378 RTC_CHECK(codec_settings->codecType == codecType_)
379 << "Unsupported codec " << codec_settings->codecType << " for "
380 << codecType_;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000381
asapersson1d61a512016-01-20 01:13:46 -0800382 codec_mode_ = codec_settings->mode;
Alex Glazneva9d08922016-02-19 15:24:06 -0800383 int init_width = codec_settings->width;
384 int init_height = codec_settings->height;
Peter Boström7ace4882016-04-14 00:54:56 +0200385 scale_ = codecType_ != kVideoCodecVP9;
Alex Glazneva9d08922016-02-19 15:24:06 -0800386
387 ALOGD << "InitEncode request: " << init_width << " x " << init_height;
Alex Glaznevfddf6e52015-10-07 16:51:02 -0700388 ALOGD << "Encoder automatic resize " << (scale_ ? "enabled" : "disabled");
Alex Glazneva9d08922016-02-19 15:24:06 -0800389
Peter Boström2bc68c72015-09-24 16:22:28 +0200390 if (scale_) {
391 if (codecType_ == kVideoCodecVP8) {
392 // QP is obtained from VP8-bitstream for HW, so the QP corresponds to the
393 // (internal) range: [0, 127]. And we cannot change QP_max in HW, so it is
394 // always = 127. Note that in SW, QP is that of the user-level range [0,
395 // 63].
Alex Glaznev79299af2016-04-12 16:39:39 -0700396 const int kLowQpThreshold = 29;
Peter Boström2c8a2962016-04-18 12:58:02 +0200397 const int kBadQpThreshold = 100;
Peter Boström3c6eac22016-04-26 13:37:10 +0200398 quality_scaler_.Init(kLowQpThreshold, kBadQpThreshold,
pboscbac40d2016-04-13 02:51:02 -0700399 codec_settings->startBitrate, codec_settings->width,
400 codec_settings->height,
401 codec_settings->maxFramerate);
Peter Boström2bc68c72015-09-24 16:22:28 +0200402 } else if (codecType_ == kVideoCodecH264) {
403 // H264 QP is in the range [0, 51].
Peter Boström2c8a2962016-04-18 12:58:02 +0200404 const int kLowQpThreshold = 24;
405 const int kBadQpThreshold = 39;
Peter Boström3c6eac22016-04-26 13:37:10 +0200406 quality_scaler_.Init(kLowQpThreshold, kBadQpThreshold,
pboscbac40d2016-04-13 02:51:02 -0700407 codec_settings->startBitrate, codec_settings->width,
408 codec_settings->height,
409 codec_settings->maxFramerate);
Peter Boström2bc68c72015-09-24 16:22:28 +0200410 } else {
411 // When adding codec support to additional hardware codecs, also configure
412 // their QP thresholds for scaling.
413 RTC_NOTREACHED() << "Unsupported codec without configured QP thresholds.";
Alex Glazneva9d08922016-02-19 15:24:06 -0800414 scale_ = false;
Peter Boström2bc68c72015-09-24 16:22:28 +0200415 }
Alex Glazneva9d08922016-02-19 15:24:06 -0800416 QualityScaler::Resolution res = quality_scaler_.GetScaledResolution();
Peter Boström926dfcd2016-04-14 14:48:10 +0200417 init_width = res.width;
418 init_height = res.height;
Alex Glazneva9d08922016-02-19 15:24:06 -0800419 ALOGD << "Scaled resolution: " << init_width << " x " << init_height;
jackychen61b4d512015-04-21 15:30:11 -0700420 }
Alex Glazneva9d08922016-02-19 15:24:06 -0800421
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000422 return codec_thread_->Invoke<int32_t>(
423 Bind(&MediaCodecVideoEncoder::InitEncodeOnCodecThread,
424 this,
Alex Glazneva9d08922016-02-19 15:24:06 -0800425 init_width,
426 init_height,
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000427 codec_settings->startBitrate,
perkj30e91822015-11-20 01:31:25 -0800428 codec_settings->maxFramerate,
429 false /* use_surface */));
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000430}
431
432int32_t MediaCodecVideoEncoder::Encode(
Miguel Casas-Sanchez47650702015-05-29 17:21:40 -0700433 const webrtc::VideoFrame& frame,
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000434 const webrtc::CodecSpecificInfo* /* codec_specific_info */,
pbos22993e12015-10-19 02:39:06 -0700435 const std::vector<webrtc::FrameType>* frame_types) {
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000436 return codec_thread_->Invoke<int32_t>(Bind(
437 &MediaCodecVideoEncoder::EncodeOnCodecThread, this, frame, frame_types));
438}
439
440int32_t MediaCodecVideoEncoder::RegisterEncodeCompleteCallback(
441 webrtc::EncodedImageCallback* callback) {
442 return codec_thread_->Invoke<int32_t>(
443 Bind(&MediaCodecVideoEncoder::RegisterEncodeCompleteCallbackOnCodecThread,
444 this,
445 callback));
446}
447
448int32_t MediaCodecVideoEncoder::Release() {
Alex Glaznevfddf6e52015-10-07 16:51:02 -0700449 ALOGD << "EncoderRelease request";
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000450 return codec_thread_->Invoke<int32_t>(
451 Bind(&MediaCodecVideoEncoder::ReleaseOnCodecThread, this));
452}
453
454int32_t MediaCodecVideoEncoder::SetChannelParameters(uint32_t /* packet_loss */,
455 int64_t /* rtt */) {
456 return WEBRTC_VIDEO_CODEC_OK;
457}
458
459int32_t MediaCodecVideoEncoder::SetRates(uint32_t new_bit_rate,
460 uint32_t frame_rate) {
461 return codec_thread_->Invoke<int32_t>(
462 Bind(&MediaCodecVideoEncoder::SetRatesOnCodecThread,
463 this,
464 new_bit_rate,
465 frame_rate));
466}
467
468void MediaCodecVideoEncoder::OnMessage(rtc::Message* msg) {
perkj9576e542015-11-12 06:43:16 -0800469 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000470 JNIEnv* jni = AttachCurrentThreadIfNeeded();
471 ScopedLocalRefFrame local_ref_frame(jni);
472
473 // We only ever send one message to |this| directly (not through a Bind()'d
474 // functor), so expect no ID/data.
henrikg91d6ede2015-09-17 00:24:34 -0700475 RTC_CHECK(!msg->message_id) << "Unexpected message!";
476 RTC_CHECK(!msg->pdata) << "Unexpected message!";
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000477 if (!inited_) {
478 return;
479 }
480
481 // It would be nice to recover from a failure here if one happened, but it's
482 // unclear how to signal such a failure to the app, so instead we stay silent
483 // about it and let the next app-called API method reveal the borkedness.
484 DeliverPendingOutputs(jni);
485 codec_thread_->PostDelayed(kMediaCodecPollMs, this);
486}
487
perkj9576e542015-11-12 06:43:16 -0800488bool MediaCodecVideoEncoder::ResetCodecOnCodecThread() {
489 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
490 ALOGE << "ResetOnCodecThread";
491 if (ReleaseOnCodecThread() != WEBRTC_VIDEO_CODEC_OK ||
perkj30e91822015-11-20 01:31:25 -0800492 InitEncodeOnCodecThread(width_, height_, 0, 0, false) !=
493 WEBRTC_VIDEO_CODEC_OK) {
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000494 // TODO(fischman): wouldn't it be nice if there was a way to gracefully
495 // degrade to a SW encoder at this point? There isn't one AFAICT :(
496 // https://code.google.com/p/webrtc/issues/detail?id=2920
perkj9576e542015-11-12 06:43:16 -0800497 return false;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000498 }
perkj9576e542015-11-12 06:43:16 -0800499 return true;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000500}
501
502int32_t MediaCodecVideoEncoder::InitEncodeOnCodecThread(
perkj30e91822015-11-20 01:31:25 -0800503 int width, int height, int kbps, int fps, bool use_surface) {
perkj9576e542015-11-12 06:43:16 -0800504 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
perkj30e91822015-11-20 01:31:25 -0800505 RTC_CHECK(!use_surface || egl_context_ != nullptr) << "EGL context not set.";
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000506 JNIEnv* jni = AttachCurrentThreadIfNeeded();
507 ScopedLocalRefFrame local_ref_frame(jni);
508
Alex Glaznevfddf6e52015-10-07 16:51:02 -0700509 ALOGD << "InitEncodeOnCodecThread Type: " << (int)codecType_ << ", " <<
510 width << " x " << height << ". Bitrate: " << kbps <<
511 " kbps. Fps: " << fps;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000512 if (kbps == 0) {
513 kbps = last_set_bitrate_kbps_;
514 }
515 if (fps == 0) {
glaznevf4decb52016-01-15 13:49:22 -0800516 fps = MAX_VIDEO_FPS;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000517 }
518
519 width_ = width;
520 height_ = height;
521 last_set_bitrate_kbps_ = kbps;
glaznevf4decb52016-01-15 13:49:22 -0800522 last_set_fps_ = (fps < MAX_VIDEO_FPS) ? fps : MAX_VIDEO_FPS;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000523 yuv_size_ = width_ * height_ * 3 / 2;
524 frames_received_ = 0;
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000525 frames_encoded_ = 0;
glaznev919ff752016-01-27 15:01:03 -0800526 frames_dropped_media_encoder_ = 0;
527 consecutive_full_queue_frame_drops_ = 0;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000528 current_timestamp_us_ = 0;
glaznev94291482016-02-01 13:17:18 -0800529 stat_start_time_ms_ = GetCurrentTimeMs();
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000530 current_frames_ = 0;
531 current_bytes_ = 0;
glaznevf4decb52016-01-15 13:49:22 -0800532 current_acc_qp_ = 0;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000533 current_encoding_time_ms_ = 0;
534 last_input_timestamp_ms_ = -1;
535 last_output_timestamp_ms_ = -1;
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000536 output_timestamp_ = 0;
537 output_render_time_ms_ = 0;
Perba7dc722016-04-19 15:01:23 +0200538 input_frame_infos_.clear();
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000539 drop_next_input_frame_ = false;
perkj30e91822015-11-20 01:31:25 -0800540 use_surface_ = use_surface;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000541 picture_id_ = static_cast<uint16_t>(rand()) & 0x7FFF;
Alex Glaznevad948c42015-11-18 13:06:42 -0800542 gof_.SetGofInfoVP9(webrtc::TemporalStructureMode::kTemporalStructureMode1);
543 tl0_pic_idx_ = static_cast<uint8_t>(rand());
544 gof_idx_ = 0;
asapersson1d61a512016-01-20 01:13:46 -0800545 last_frame_received_ms_ = -1;
546 frames_received_since_last_key_ = kMinKeyFrameInterval;
perkj9576e542015-11-12 06:43:16 -0800547
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000548 // We enforce no extra stride/padding in the format creation step.
Perec2922f2016-01-27 15:25:46 +0100549 jobject j_video_codec_enum = JavaEnumFromIndexAndClassName(
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000550 jni, "MediaCodecVideoEncoder$VideoCodecType", codecType_);
perkj9576e542015-11-12 06:43:16 -0800551 const bool encode_status = jni->CallBooleanMethod(
552 *j_media_codec_video_encoder_, j_init_encode_method_,
perkj30e91822015-11-20 01:31:25 -0800553 j_video_codec_enum, width, height, kbps, fps,
554 (use_surface ? egl_context_ : nullptr));
perkj9576e542015-11-12 06:43:16 -0800555 if (!encode_status) {
556 ALOGE << "Failed to configure encoder.";
557 return WEBRTC_VIDEO_CODEC_ERROR;
558 }
559 CHECK_EXCEPTION(jni);
560
Per598242a2015-11-26 14:28:55 +0100561 if (!use_surface) {
perkj30e91822015-11-20 01:31:25 -0800562 jobjectArray input_buffers = reinterpret_cast<jobjectArray>(
563 jni->CallObjectMethod(*j_media_codec_video_encoder_,
564 j_get_input_buffers_method_));
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000565 CHECK_EXCEPTION(jni);
perkj30e91822015-11-20 01:31:25 -0800566 if (IsNull(jni, input_buffers)) {
567 return WEBRTC_VIDEO_CODEC_ERROR;
568 }
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000569
perkj30e91822015-11-20 01:31:25 -0800570 switch (GetIntField(jni, *j_media_codec_video_encoder_,
571 j_color_format_field_)) {
572 case COLOR_FormatYUV420Planar:
573 encoder_fourcc_ = libyuv::FOURCC_YU12;
574 break;
575 case COLOR_FormatYUV420SemiPlanar:
576 case COLOR_QCOM_FormatYUV420SemiPlanar:
577 case COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m:
578 encoder_fourcc_ = libyuv::FOURCC_NV12;
579 break;
580 default:
581 LOG(LS_ERROR) << "Wrong color format.";
582 return WEBRTC_VIDEO_CODEC_ERROR;
583 }
584 size_t num_input_buffers = jni->GetArrayLength(input_buffers);
585 RTC_CHECK(input_buffers_.empty())
586 << "Unexpected double InitEncode without Release";
587 input_buffers_.resize(num_input_buffers);
588 for (size_t i = 0; i < num_input_buffers; ++i) {
589 input_buffers_[i] =
590 jni->NewGlobalRef(jni->GetObjectArrayElement(input_buffers, i));
591 int64_t yuv_buffer_capacity =
592 jni->GetDirectBufferCapacity(input_buffers_[i]);
593 CHECK_EXCEPTION(jni);
594 RTC_CHECK(yuv_buffer_capacity >= yuv_size_) << "Insufficient capacity";
595 }
596 }
perkj9576e542015-11-12 06:43:16 -0800597
598 inited_ = true;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000599 codec_thread_->PostDelayed(kMediaCodecPollMs, this);
600 return WEBRTC_VIDEO_CODEC_OK;
601}
602
603int32_t MediaCodecVideoEncoder::EncodeOnCodecThread(
Miguel Casas-Sanchez47650702015-05-29 17:21:40 -0700604 const webrtc::VideoFrame& frame,
pbos22993e12015-10-19 02:39:06 -0700605 const std::vector<webrtc::FrameType>* frame_types) {
perkj9576e542015-11-12 06:43:16 -0800606 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000607 JNIEnv* jni = AttachCurrentThreadIfNeeded();
608 ScopedLocalRefFrame local_ref_frame(jni);
609
610 if (!inited_) {
611 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
612 }
perkj9576e542015-11-12 06:43:16 -0800613
asapersson1d61a512016-01-20 01:13:46 -0800614 bool send_key_frame = false;
glaznev94291482016-02-01 13:17:18 -0800615 if (codec_mode_ == webrtc::kRealtimeVideo) {
asapersson1d61a512016-01-20 01:13:46 -0800616 ++frames_received_since_last_key_;
617 int64_t now_ms = GetCurrentTimeMs();
618 if (last_frame_received_ms_ != -1 &&
619 (now_ms - last_frame_received_ms_) > kFrameDiffThresholdMs) {
620 // Add limit to prevent triggering a key for every frame for very low
621 // framerates (e.g. if frame diff > kFrameDiffThresholdMs).
622 if (frames_received_since_last_key_ > kMinKeyFrameInterval) {
623 ALOGD << "Send key, frame diff: " << (now_ms - last_frame_received_ms_);
624 send_key_frame = true;
625 }
626 frames_received_since_last_key_ = 0;
627 }
628 last_frame_received_ms_ = now_ms;
629 }
630
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000631 frames_received_++;
632 if (!DeliverPendingOutputs(jni)) {
perkj9576e542015-11-12 06:43:16 -0800633 if (!ResetCodecOnCodecThread())
634 return WEBRTC_VIDEO_CODEC_ERROR;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000635 }
glaznevf4decb52016-01-15 13:49:22 -0800636 if (frames_encoded_ < kMaxEncodedLogFrames) {
Perba7dc722016-04-19 15:01:23 +0200637 ALOGD << "Encoder frame in # " << (frames_received_ - 1)
638 << ". TS: " << (int)(current_timestamp_us_ / 1000)
639 << ". Q: " << input_frame_infos_.size() << ". Fps: " << last_set_fps_
640 << ". Kbps: " << last_set_bitrate_kbps_;
glaznevf4decb52016-01-15 13:49:22 -0800641 }
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000642
643 if (drop_next_input_frame_) {
perkj9576e542015-11-12 06:43:16 -0800644 ALOGW << "Encoder drop frame - failed callback.";
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000645 drop_next_input_frame_ = false;
glaznevf4decb52016-01-15 13:49:22 -0800646 current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
glaznev919ff752016-01-27 15:01:03 -0800647 frames_dropped_media_encoder_++;
glaznevf4decb52016-01-15 13:49:22 -0800648 OnDroppedFrame();
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000649 return WEBRTC_VIDEO_CODEC_OK;
650 }
651
henrikg91d6ede2015-09-17 00:24:34 -0700652 RTC_CHECK(frame_types->size() == 1) << "Unexpected stream count";
Peter Boström2bc68c72015-09-24 16:22:28 +0200653
Peter Boströmf7704d12016-04-11 16:42:40 +0200654 // Check if we accumulated too many frames in encoder input buffers and drop
655 // frame if so.
Perba7dc722016-04-19 15:01:23 +0200656 if (input_frame_infos_.size() > MAX_ENCODER_Q_SIZE) {
657 ALOGD << "Already " << input_frame_infos_.size()
658 << " frames in the queue, dropping"
Peter Boströmf7704d12016-04-11 16:42:40 +0200659 << ". TS: " << (int)(current_timestamp_us_ / 1000)
660 << ". Fps: " << last_set_fps_
661 << ". Consecutive drops: " << consecutive_full_queue_frame_drops_;
662 current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
663 consecutive_full_queue_frame_drops_++;
664 if (consecutive_full_queue_frame_drops_ >=
665 ENCODER_STALL_FRAMEDROP_THRESHOLD) {
666 ALOGE << "Encoder got stuck. Reset.";
667 ResetCodecOnCodecThread();
668 return WEBRTC_VIDEO_CODEC_ERROR;
glaznevf4decb52016-01-15 13:49:22 -0800669 }
Peter Boströmf7704d12016-04-11 16:42:40 +0200670 frames_dropped_media_encoder_++;
671 OnDroppedFrame();
672 return WEBRTC_VIDEO_CODEC_OK;
glaznevf4decb52016-01-15 13:49:22 -0800673 }
glaznev919ff752016-01-27 15:01:03 -0800674 consecutive_full_queue_frame_drops_ = 0;
glaznevf4decb52016-01-15 13:49:22 -0800675
Per598242a2015-11-26 14:28:55 +0100676 VideoFrame input_frame = frame;
677 if (scale_) {
678 // Check framerate before spatial resolution change.
679 quality_scaler_.OnEncodeFrame(frame);
680 const webrtc::QualityScaler::Resolution scaled_resolution =
681 quality_scaler_.GetScaledResolution();
682 if (scaled_resolution.width != frame.width() ||
683 scaled_resolution.height != frame.height()) {
nisse26acec42016-04-15 03:43:39 -0700684 if (frame.video_frame_buffer()->native_handle() != nullptr) {
Per598242a2015-11-26 14:28:55 +0100685 rtc::scoped_refptr<webrtc::VideoFrameBuffer> scaled_buffer(
686 static_cast<AndroidTextureBuffer*>(
Per71f5a9a2015-12-11 09:32:37 +0100687 frame.video_frame_buffer().get())->ScaleAndRotate(
Per598242a2015-11-26 14:28:55 +0100688 scaled_resolution.width,
Per71f5a9a2015-12-11 09:32:37 +0100689 scaled_resolution.height,
690 webrtc::kVideoRotation_0));
Per598242a2015-11-26 14:28:55 +0100691 input_frame.set_video_frame_buffer(scaled_buffer);
692 } else {
693 input_frame = quality_scaler_.GetScaledFrame(frame);
694 }
695 }
696 }
jackychen61b4d512015-04-21 15:30:11 -0700697
perkj9576e542015-11-12 06:43:16 -0800698 if (!MaybeReconfigureEncoderOnCodecThread(input_frame)) {
699 ALOGE << "Failed to reconfigure encoder.";
700 return WEBRTC_VIDEO_CODEC_ERROR;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000701 }
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000702
Perba7dc722016-04-19 15:01:23 +0200703 const int64_t time_before_calling_encode = GetCurrentTimeMs();
asapersson1d61a512016-01-20 01:13:46 -0800704 const bool key_frame =
705 frame_types->front() != webrtc::kVideoFrameDelta || send_key_frame;
perkj30e91822015-11-20 01:31:25 -0800706 bool encode_status = true;
nisse26acec42016-04-15 03:43:39 -0700707 if (!input_frame.video_frame_buffer()->native_handle()) {
perkj30e91822015-11-20 01:31:25 -0800708 int j_input_buffer_index = jni->CallIntMethod(*j_media_codec_video_encoder_,
709 j_dequeue_input_buffer_method_);
710 CHECK_EXCEPTION(jni);
711 if (j_input_buffer_index == -1) {
712 // Video codec falls behind - no input buffer available.
713 ALOGW << "Encoder drop frame - no input buffers available";
glaznev919ff752016-01-27 15:01:03 -0800714 current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
715 frames_dropped_media_encoder_++;
716 OnDroppedFrame();
perkj30e91822015-11-20 01:31:25 -0800717 return WEBRTC_VIDEO_CODEC_OK; // TODO(fischman): see webrtc bug 2887.
718 }
719 if (j_input_buffer_index == -2) {
720 ResetCodecOnCodecThread();
721 return WEBRTC_VIDEO_CODEC_ERROR;
722 }
723 encode_status = EncodeByteBufferOnCodecThread(jni, key_frame, input_frame,
724 j_input_buffer_index);
725 } else {
726 encode_status = EncodeTextureOnCodecThread(jni, key_frame, input_frame);
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000727 }
perkj30e91822015-11-20 01:31:25 -0800728
729 if (!encode_status) {
730 ALOGE << "Failed encode frame with timestamp: " << input_frame.timestamp();
perkj9576e542015-11-12 06:43:16 -0800731 ResetCodecOnCodecThread();
perkj12f68022015-10-16 13:31:45 +0200732 return WEBRTC_VIDEO_CODEC_ERROR;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000733 }
734
Perba7dc722016-04-19 15:01:23 +0200735 // Save input image timestamps for later output.
736 input_frame_infos_.emplace_back(
737 time_before_calling_encode, input_frame.timestamp(),
738 input_frame.render_time_ms(), input_frame.rotation());
739
perkj9576e542015-11-12 06:43:16 -0800740 last_input_timestamp_ms_ =
741 current_timestamp_us_ / rtc::kNumMicrosecsPerMillisec;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000742
perkj9576e542015-11-12 06:43:16 -0800743 current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
744
perkj30e91822015-11-20 01:31:25 -0800745 if (!DeliverPendingOutputs(jni)) {
perkj9576e542015-11-12 06:43:16 -0800746 ALOGE << "Failed deliver pending outputs.";
747 ResetCodecOnCodecThread();
748 return WEBRTC_VIDEO_CODEC_ERROR;
749 }
750 return WEBRTC_VIDEO_CODEC_OK;
751}
752
753bool MediaCodecVideoEncoder::MaybeReconfigureEncoderOnCodecThread(
754 const webrtc::VideoFrame& frame) {
755 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
756
nisse26acec42016-04-15 03:43:39 -0700757 const bool is_texture_frame =
758 frame.video_frame_buffer()->native_handle() != nullptr;
perkj30e91822015-11-20 01:31:25 -0800759 const bool reconfigure_due_to_format = is_texture_frame != use_surface_;
perkj9576e542015-11-12 06:43:16 -0800760 const bool reconfigure_due_to_size =
761 frame.width() != width_ || frame.height() != height_;
762
perkj30e91822015-11-20 01:31:25 -0800763 if (reconfigure_due_to_format) {
764 ALOGD << "Reconfigure encoder due to format change. "
765 << (use_surface_ ?
766 "Reconfiguring to encode from byte buffer." :
767 "Reconfiguring to encode from texture.");
glaznev94291482016-02-01 13:17:18 -0800768 LogStatistics(true);
perkj30e91822015-11-20 01:31:25 -0800769 }
perkj9576e542015-11-12 06:43:16 -0800770 if (reconfigure_due_to_size) {
glaznev94291482016-02-01 13:17:18 -0800771 ALOGW << "Reconfigure encoder due to frame resolution change from "
perkj9576e542015-11-12 06:43:16 -0800772 << width_ << " x " << height_ << " to " << frame.width() << " x "
773 << frame.height();
glaznev94291482016-02-01 13:17:18 -0800774 LogStatistics(true);
perkj9576e542015-11-12 06:43:16 -0800775 width_ = frame.width();
776 height_ = frame.height();
777 }
778
perkj30e91822015-11-20 01:31:25 -0800779 if (!reconfigure_due_to_format && !reconfigure_due_to_size)
perkj9576e542015-11-12 06:43:16 -0800780 return true;
781
782 ReleaseOnCodecThread();
783
perkj30e91822015-11-20 01:31:25 -0800784 return InitEncodeOnCodecThread(width_, height_, 0, 0 , is_texture_frame) ==
perkj9576e542015-11-12 06:43:16 -0800785 WEBRTC_VIDEO_CODEC_OK;
786}
787
788bool MediaCodecVideoEncoder::EncodeByteBufferOnCodecThread(JNIEnv* jni,
789 bool key_frame, const webrtc::VideoFrame& frame, int input_buffer_index) {
790 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
perkj30e91822015-11-20 01:31:25 -0800791 RTC_CHECK(!use_surface_);
perkj9576e542015-11-12 06:43:16 -0800792
perkj9576e542015-11-12 06:43:16 -0800793 jobject j_input_buffer = input_buffers_[input_buffer_index];
794 uint8_t* yuv_buffer =
795 reinterpret_cast<uint8_t*>(jni->GetDirectBufferAddress(j_input_buffer));
796 CHECK_EXCEPTION(jni);
797 RTC_CHECK(yuv_buffer) << "Indirect buffer??";
798 RTC_CHECK(!libyuv::ConvertFromI420(
799 frame.buffer(webrtc::kYPlane), frame.stride(webrtc::kYPlane),
800 frame.buffer(webrtc::kUPlane), frame.stride(webrtc::kUPlane),
801 frame.buffer(webrtc::kVPlane), frame.stride(webrtc::kVPlane),
802 yuv_buffer, width_, width_, height_, encoder_fourcc_))
803 << "ConvertFromI420 failed";
804
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000805 bool encode_status = jni->CallBooleanMethod(*j_media_codec_video_encoder_,
perkj9576e542015-11-12 06:43:16 -0800806 j_encode_buffer_method_,
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000807 key_frame,
perkj9576e542015-11-12 06:43:16 -0800808 input_buffer_index,
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000809 yuv_size_,
810 current_timestamp_us_);
811 CHECK_EXCEPTION(jni);
perkj9576e542015-11-12 06:43:16 -0800812 return encode_status;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000813}
814
perkj30e91822015-11-20 01:31:25 -0800815bool MediaCodecVideoEncoder::EncodeTextureOnCodecThread(JNIEnv* jni,
816 bool key_frame, const webrtc::VideoFrame& frame) {
817 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
818 RTC_CHECK(use_surface_);
nisse26acec42016-04-15 03:43:39 -0700819 NativeHandleImpl* handle = static_cast<NativeHandleImpl*>(
820 frame.video_frame_buffer()->native_handle());
perkj30e91822015-11-20 01:31:25 -0800821 jfloatArray sampling_matrix = jni->NewFloatArray(16);
822 jni->SetFloatArrayRegion(sampling_matrix, 0, 16, handle->sampling_matrix);
823
824 bool encode_status = jni->CallBooleanMethod(*j_media_codec_video_encoder_,
825 j_encode_texture_method_,
826 key_frame,
827 handle->oes_texture_id,
828 sampling_matrix,
829 current_timestamp_us_);
830 CHECK_EXCEPTION(jni);
831 return encode_status;
832}
833
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000834int32_t MediaCodecVideoEncoder::RegisterEncodeCompleteCallbackOnCodecThread(
835 webrtc::EncodedImageCallback* callback) {
perkj9576e542015-11-12 06:43:16 -0800836 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000837 JNIEnv* jni = AttachCurrentThreadIfNeeded();
838 ScopedLocalRefFrame local_ref_frame(jni);
839 callback_ = callback;
840 return WEBRTC_VIDEO_CODEC_OK;
841}
842
843int32_t MediaCodecVideoEncoder::ReleaseOnCodecThread() {
perkj9576e542015-11-12 06:43:16 -0800844 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000845 if (!inited_) {
846 return WEBRTC_VIDEO_CODEC_OK;
847 }
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000848 JNIEnv* jni = AttachCurrentThreadIfNeeded();
Alex Glaznevfddf6e52015-10-07 16:51:02 -0700849 ALOGD << "EncoderReleaseOnCodecThread: Frames received: " <<
850 frames_received_ << ". Encoded: " << frames_encoded_ <<
glaznev919ff752016-01-27 15:01:03 -0800851 ". Dropped: " << frames_dropped_media_encoder_;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000852 ScopedLocalRefFrame local_ref_frame(jni);
853 for (size_t i = 0; i < input_buffers_.size(); ++i)
854 jni->DeleteGlobalRef(input_buffers_[i]);
855 input_buffers_.clear();
856 jni->CallVoidMethod(*j_media_codec_video_encoder_, j_release_method_);
857 CHECK_EXCEPTION(jni);
858 rtc::MessageQueueManager::Clear(this);
859 inited_ = false;
perkj30e91822015-11-20 01:31:25 -0800860 use_surface_ = false;
Alex Glaznevfddf6e52015-10-07 16:51:02 -0700861 ALOGD << "EncoderReleaseOnCodecThread done.";
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000862 return WEBRTC_VIDEO_CODEC_OK;
863}
864
865int32_t MediaCodecVideoEncoder::SetRatesOnCodecThread(uint32_t new_bit_rate,
866 uint32_t frame_rate) {
perkj9576e542015-11-12 06:43:16 -0800867 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
glaznevf4decb52016-01-15 13:49:22 -0800868 frame_rate = (frame_rate < MAX_ALLOWED_VIDEO_FPS) ?
869 frame_rate : MAX_ALLOWED_VIDEO_FPS;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000870 if (last_set_bitrate_kbps_ == new_bit_rate &&
871 last_set_fps_ == frame_rate) {
872 return WEBRTC_VIDEO_CODEC_OK;
873 }
glaznev919ff752016-01-27 15:01:03 -0800874 if (scale_) {
875 quality_scaler_.ReportFramerate(frame_rate);
876 }
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000877 JNIEnv* jni = AttachCurrentThreadIfNeeded();
878 ScopedLocalRefFrame local_ref_frame(jni);
879 if (new_bit_rate > 0) {
880 last_set_bitrate_kbps_ = new_bit_rate;
881 }
882 if (frame_rate > 0) {
883 last_set_fps_ = frame_rate;
884 }
885 bool ret = jni->CallBooleanMethod(*j_media_codec_video_encoder_,
886 j_set_rates_method_,
887 last_set_bitrate_kbps_,
888 last_set_fps_);
889 CHECK_EXCEPTION(jni);
890 if (!ret) {
perkj9576e542015-11-12 06:43:16 -0800891 ResetCodecOnCodecThread();
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000892 return WEBRTC_VIDEO_CODEC_ERROR;
893 }
894 return WEBRTC_VIDEO_CODEC_OK;
895}
896
897int MediaCodecVideoEncoder::GetOutputBufferInfoIndex(
898 JNIEnv* jni,
899 jobject j_output_buffer_info) {
900 return GetIntField(jni, j_output_buffer_info, j_info_index_field_);
901}
902
903jobject MediaCodecVideoEncoder::GetOutputBufferInfoBuffer(
904 JNIEnv* jni,
905 jobject j_output_buffer_info) {
906 return GetObjectField(jni, j_output_buffer_info, j_info_buffer_field_);
907}
908
909bool MediaCodecVideoEncoder::GetOutputBufferInfoIsKeyFrame(
910 JNIEnv* jni,
911 jobject j_output_buffer_info) {
912 return GetBooleanField(jni, j_output_buffer_info, j_info_is_key_frame_field_);
913}
914
915jlong MediaCodecVideoEncoder::GetOutputBufferInfoPresentationTimestampUs(
916 JNIEnv* jni,
917 jobject j_output_buffer_info) {
918 return GetLongField(
919 jni, j_output_buffer_info, j_info_presentation_timestamp_us_field_);
920}
921
922bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) {
perkj9576e542015-11-12 06:43:16 -0800923 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000924 while (true) {
925 jobject j_output_buffer_info = jni->CallObjectMethod(
926 *j_media_codec_video_encoder_, j_dequeue_output_buffer_method_);
927 CHECK_EXCEPTION(jni);
928 if (IsNull(jni, j_output_buffer_info)) {
929 break;
930 }
931
932 int output_buffer_index =
933 GetOutputBufferInfoIndex(jni, j_output_buffer_info);
934 if (output_buffer_index == -1) {
perkj9576e542015-11-12 06:43:16 -0800935 ResetCodecOnCodecThread();
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000936 return false;
937 }
938
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000939 // Get key and config frame flags.
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000940 jobject j_output_buffer =
941 GetOutputBufferInfoBuffer(jni, j_output_buffer_info);
942 bool key_frame = GetOutputBufferInfoIsKeyFrame(jni, j_output_buffer_info);
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000943
944 // Get frame timestamps from a queue - for non config frames only.
945 int64_t frame_encoding_time_ms = 0;
946 last_output_timestamp_ms_ =
947 GetOutputBufferInfoPresentationTimestampUs(jni, j_output_buffer_info) /
948 1000;
Perba7dc722016-04-19 15:01:23 +0200949 if (!input_frame_infos_.empty()) {
950 const InputFrameInfo& frame_info = input_frame_infos_.front();
951 output_timestamp_ = frame_info.frame_timestamp;
952 output_render_time_ms_ = frame_info.frame_render_time_ms;
953 output_rotation_ = frame_info.rotation;
954 frame_encoding_time_ms =
955 GetCurrentTimeMs() - frame_info.encode_start_time;
956 input_frame_infos_.pop_front();
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000957 }
958
959 // Extract payload.
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000960 size_t payload_size = jni->GetDirectBufferCapacity(j_output_buffer);
Peter Boström0c4e06b2015-10-07 12:23:21 +0200961 uint8_t* payload = reinterpret_cast<uint8_t*>(
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000962 jni->GetDirectBufferAddress(j_output_buffer));
963 CHECK_EXCEPTION(jni);
964
glaznevf4decb52016-01-15 13:49:22 -0800965 if (frames_encoded_ < kMaxEncodedLogFrames) {
glaznev94291482016-02-01 13:17:18 -0800966 int current_latency =
967 (int)(last_input_timestamp_ms_ - last_output_timestamp_ms_);
968 ALOGD << "Encoder frame out # " << frames_encoded_ <<
969 ". Key: " << key_frame <<
970 ". Size: " << payload_size <<
971 ". TS: " << (int)last_output_timestamp_ms_ <<
972 ". Latency: " << current_latency <<
glaznevf4decb52016-01-15 13:49:22 -0800973 ". EncTime: " << frame_encoding_time_ms;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000974 }
975
976 // Callback - return encoded frame.
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000977 int32_t callback_status = 0;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000978 if (callback_) {
979 scoped_ptr<webrtc::EncodedImage> image(
980 new webrtc::EncodedImage(payload, payload_size, payload_size));
981 image->_encodedWidth = width_;
982 image->_encodedHeight = height_;
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000983 image->_timeStamp = output_timestamp_;
984 image->capture_time_ms_ = output_render_time_ms_;
Perba7dc722016-04-19 15:01:23 +0200985 image->rotation_ = output_rotation_;
Peter Boström49e196a2015-10-23 15:58:18 +0200986 image->_frameType =
987 (key_frame ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta);
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000988 image->_completeFrame = true;
asapersson075fb4b2015-10-29 08:49:14 -0700989 image->adapt_reason_.quality_resolution_downscales =
990 scale_ ? quality_scaler_.downscale_shift() : -1;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000991
992 webrtc::CodecSpecificInfo info;
993 memset(&info, 0, sizeof(info));
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000994 info.codecType = codecType_;
995 if (codecType_ == kVideoCodecVP8) {
996 info.codecSpecific.VP8.pictureId = picture_id_;
997 info.codecSpecific.VP8.nonReference = false;
998 info.codecSpecific.VP8.simulcastIdx = 0;
999 info.codecSpecific.VP8.temporalIdx = webrtc::kNoTemporalIdx;
1000 info.codecSpecific.VP8.layerSync = false;
1001 info.codecSpecific.VP8.tl0PicIdx = webrtc::kNoTl0PicIdx;
1002 info.codecSpecific.VP8.keyIdx = webrtc::kNoKeyIdx;
Alex Glaznevad948c42015-11-18 13:06:42 -08001003 } else if (codecType_ == kVideoCodecVP9) {
1004 if (key_frame) {
1005 gof_idx_ = 0;
1006 }
1007 info.codecSpecific.VP9.picture_id = picture_id_;
1008 info.codecSpecific.VP9.inter_pic_predicted = key_frame ? false : true;
1009 info.codecSpecific.VP9.flexible_mode = false;
1010 info.codecSpecific.VP9.ss_data_available = key_frame ? true : false;
1011 info.codecSpecific.VP9.tl0_pic_idx = tl0_pic_idx_++;
1012 info.codecSpecific.VP9.temporal_idx = webrtc::kNoTemporalIdx;
1013 info.codecSpecific.VP9.spatial_idx = webrtc::kNoSpatialIdx;
1014 info.codecSpecific.VP9.temporal_up_switch = true;
1015 info.codecSpecific.VP9.inter_layer_predicted = false;
1016 info.codecSpecific.VP9.gof_idx =
1017 static_cast<uint8_t>(gof_idx_++ % gof_.num_frames_in_gof);
1018 info.codecSpecific.VP9.num_spatial_layers = 1;
1019 info.codecSpecific.VP9.spatial_layer_resolution_present = false;
1020 if (info.codecSpecific.VP9.ss_data_available) {
1021 info.codecSpecific.VP9.spatial_layer_resolution_present = true;
1022 info.codecSpecific.VP9.width[0] = width_;
1023 info.codecSpecific.VP9.height[0] = height_;
1024 info.codecSpecific.VP9.gof.CopyGofInfoVP9(gof_);
1025 }
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001026 }
Alex Glaznevad948c42015-11-18 13:06:42 -08001027 picture_id_ = (picture_id_ + 1) & 0x7FFF;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001028
1029 // Generate a header describing a single fragment.
1030 webrtc::RTPFragmentationHeader header;
1031 memset(&header, 0, sizeof(header));
Alex Glaznevad948c42015-11-18 13:06:42 -08001032 if (codecType_ == kVideoCodecVP8 || codecType_ == kVideoCodecVP9) {
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001033 header.VerifyAndAllocateFragmentationHeader(1);
1034 header.fragmentationOffset[0] = 0;
1035 header.fragmentationLength[0] = image->_length;
1036 header.fragmentationPlType[0] = 0;
1037 header.fragmentationTimeDiff[0] = 0;
Alex Glaznevad948c42015-11-18 13:06:42 -08001038 if (codecType_ == kVideoCodecVP8 && scale_) {
asapersson86b01602015-10-20 23:55:26 -07001039 int qp;
glaznevf4decb52016-01-15 13:49:22 -08001040 if (webrtc::vp8::GetQp(payload, payload_size, &qp)) {
1041 current_acc_qp_ += qp;
asapersson86b01602015-10-20 23:55:26 -07001042 quality_scaler_.ReportQP(qp);
asapersson24ebc442016-04-19 23:48:21 -07001043 image->qp_ = qp;
glaznevf4decb52016-01-15 13:49:22 -08001044 }
asapersson86b01602015-10-20 23:55:26 -07001045 }
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001046 } else if (codecType_ == kVideoCodecH264) {
Peter Boström2bc68c72015-09-24 16:22:28 +02001047 if (scale_) {
1048 h264_bitstream_parser_.ParseBitstream(payload, payload_size);
1049 int qp;
glaznevf4decb52016-01-15 13:49:22 -08001050 if (h264_bitstream_parser_.GetLastSliceQp(&qp)) {
1051 current_acc_qp_ += qp;
Peter Boström2bc68c72015-09-24 16:22:28 +02001052 quality_scaler_.ReportQP(qp);
glaznevf4decb52016-01-15 13:49:22 -08001053 }
Peter Boström2bc68c72015-09-24 16:22:28 +02001054 }
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001055 // For H.264 search for start codes.
1056 int32_t scPositions[MAX_NALUS_PERFRAME + 1] = {};
1057 int32_t scPositionsLength = 0;
1058 int32_t scPosition = 0;
1059 while (scPositionsLength < MAX_NALUS_PERFRAME) {
1060 int32_t naluPosition = NextNaluPosition(
1061 payload + scPosition, payload_size - scPosition);
1062 if (naluPosition < 0) {
1063 break;
1064 }
1065 scPosition += naluPosition;
1066 scPositions[scPositionsLength++] = scPosition;
1067 scPosition += H264_SC_LENGTH;
1068 }
1069 if (scPositionsLength == 0) {
Alex Glaznevfddf6e52015-10-07 16:51:02 -07001070 ALOGE << "Start code is not found!";
1071 ALOGE << "Data:" << image->_buffer[0] << " " << image->_buffer[1]
1072 << " " << image->_buffer[2] << " " << image->_buffer[3]
1073 << " " << image->_buffer[4] << " " << image->_buffer[5];
perkj9576e542015-11-12 06:43:16 -08001074 ResetCodecOnCodecThread();
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001075 return false;
1076 }
1077 scPositions[scPositionsLength] = payload_size;
1078 header.VerifyAndAllocateFragmentationHeader(scPositionsLength);
1079 for (size_t i = 0; i < scPositionsLength; i++) {
1080 header.fragmentationOffset[i] = scPositions[i] + H264_SC_LENGTH;
1081 header.fragmentationLength[i] =
1082 scPositions[i + 1] - header.fragmentationOffset[i];
1083 header.fragmentationPlType[i] = 0;
1084 header.fragmentationTimeDiff[i] = 0;
1085 }
1086 }
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001087
1088 callback_status = callback_->Encoded(*image, &info, &header);
1089 }
1090
1091 // Return output buffer back to the encoder.
1092 bool success = jni->CallBooleanMethod(*j_media_codec_video_encoder_,
1093 j_release_output_buffer_method_,
1094 output_buffer_index);
1095 CHECK_EXCEPTION(jni);
1096 if (!success) {
perkj9576e542015-11-12 06:43:16 -08001097 ResetCodecOnCodecThread();
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001098 return false;
1099 }
1100
glaznevf4decb52016-01-15 13:49:22 -08001101 // Calculate and print encoding statistics - every 3 seconds.
1102 frames_encoded_++;
1103 current_frames_++;
1104 current_bytes_ += payload_size;
1105 current_encoding_time_ms_ += frame_encoding_time_ms;
glaznev94291482016-02-01 13:17:18 -08001106 LogStatistics(false);
glaznevf4decb52016-01-15 13:49:22 -08001107
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001108 if (callback_status > 0) {
1109 drop_next_input_frame_ = true;
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001110 // Theoretically could handle callback_status<0 here, but unclear what
1111 // that would mean for us.
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001112 }
1113 }
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001114 return true;
1115}
1116
glaznev94291482016-02-01 13:17:18 -08001117void MediaCodecVideoEncoder::LogStatistics(bool force_log) {
1118 int statistic_time_ms = GetCurrentTimeMs() - stat_start_time_ms_;
1119 if ((statistic_time_ms >= kMediaCodecStatisticsIntervalMs || force_log) &&
1120 current_frames_ > 0 && statistic_time_ms > 0) {
1121 int current_bitrate = current_bytes_ * 8 / statistic_time_ms;
1122 int current_fps =
1123 (current_frames_ * 1000 + statistic_time_ms / 2) / statistic_time_ms;
1124 ALOGD << "Encoded frames: " << frames_encoded_ <<
1125 ". Bitrate: " << current_bitrate <<
1126 ", target: " << last_set_bitrate_kbps_ << " kbps" <<
1127 ", fps: " << current_fps <<
1128 ", encTime: " << (current_encoding_time_ms_ / current_frames_) <<
1129 ". QP: " << (current_acc_qp_ / current_frames_) <<
1130 " for last " << statistic_time_ms << " ms.";
1131 stat_start_time_ms_ = GetCurrentTimeMs();
1132 current_frames_ = 0;
1133 current_bytes_ = 0;
1134 current_acc_qp_ = 0;
1135 current_encoding_time_ms_ = 0;
1136 }
1137}
1138
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001139int32_t MediaCodecVideoEncoder::NextNaluPosition(
1140 uint8_t *buffer, size_t buffer_size) {
1141 if (buffer_size < H264_SC_LENGTH) {
1142 return -1;
1143 }
1144 uint8_t *head = buffer;
1145 // Set end buffer pointer to 4 bytes before actual buffer end so we can
1146 // access head[1], head[2] and head[3] in a loop without buffer overrun.
1147 uint8_t *end = buffer + buffer_size - H264_SC_LENGTH;
1148
1149 while (head < end) {
1150 if (head[0]) {
1151 head++;
1152 continue;
1153 }
1154 if (head[1]) { // got 00xx
1155 head += 2;
1156 continue;
1157 }
1158 if (head[2]) { // got 0000xx
1159 head += 3;
1160 continue;
1161 }
1162 if (head[3] != 0x01) { // got 000000xx
glaznev@webrtc.orgdc08a232015-03-06 23:32:20 +00001163 head++; // xx != 1, continue searching.
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001164 continue;
1165 }
1166 return (int32_t)(head - buffer);
1167 }
1168 return -1;
1169}
1170
jackychen61b4d512015-04-21 15:30:11 -07001171void MediaCodecVideoEncoder::OnDroppedFrame() {
glaznevf4decb52016-01-15 13:49:22 -08001172 // Report dropped frame to quality_scaler_.
Peter Boström2bc68c72015-09-24 16:22:28 +02001173 if (scale_)
1174 quality_scaler_.ReportDroppedFrame();
jackychen61b4d512015-04-21 15:30:11 -07001175}
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001176
Peter Boströmb7d9a972015-12-18 16:01:11 +01001177const char* MediaCodecVideoEncoder::ImplementationName() const {
1178 return "MediaCodec";
1179}
1180
perkj461121c2016-02-15 06:28:36 -08001181MediaCodecVideoEncoderFactory::MediaCodecVideoEncoderFactory()
1182 : egl_context_(nullptr) {
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001183 JNIEnv* jni = AttachCurrentThreadIfNeeded();
1184 ScopedLocalRefFrame local_ref_frame(jni);
1185 jclass j_encoder_class = FindClass(jni, "org/webrtc/MediaCodecVideoEncoder");
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001186 supported_codecs_.clear();
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001187
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001188 bool is_vp8_hw_supported = jni->CallStaticBooleanMethod(
1189 j_encoder_class,
1190 GetStaticMethodID(jni, j_encoder_class, "isVp8HwSupported", "()Z"));
1191 CHECK_EXCEPTION(jni);
1192 if (is_vp8_hw_supported) {
Alex Glaznevfddf6e52015-10-07 16:51:02 -07001193 ALOGD << "VP8 HW Encoder supported.";
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001194 supported_codecs_.push_back(VideoCodec(kVideoCodecVP8, "VP8",
1195 MAX_VIDEO_WIDTH, MAX_VIDEO_HEIGHT, MAX_VIDEO_FPS));
1196 }
1197
Alex Glaznevad948c42015-11-18 13:06:42 -08001198 bool is_vp9_hw_supported = jni->CallStaticBooleanMethod(
1199 j_encoder_class,
1200 GetStaticMethodID(jni, j_encoder_class, "isVp9HwSupported", "()Z"));
1201 CHECK_EXCEPTION(jni);
1202 if (is_vp9_hw_supported) {
1203 ALOGD << "VP9 HW Encoder supported.";
1204 supported_codecs_.push_back(VideoCodec(kVideoCodecVP9, "VP9",
1205 MAX_VIDEO_WIDTH, MAX_VIDEO_HEIGHT, MAX_VIDEO_FPS));
1206 }
1207
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001208 bool is_h264_hw_supported = jni->CallStaticBooleanMethod(
1209 j_encoder_class,
1210 GetStaticMethodID(jni, j_encoder_class, "isH264HwSupported", "()Z"));
1211 CHECK_EXCEPTION(jni);
1212 if (is_h264_hw_supported) {
Alex Glaznevfddf6e52015-10-07 16:51:02 -07001213 ALOGD << "H.264 HW Encoder supported.";
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001214 supported_codecs_.push_back(VideoCodec(kVideoCodecH264, "H264",
1215 MAX_VIDEO_WIDTH, MAX_VIDEO_HEIGHT, MAX_VIDEO_FPS));
1216 }
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001217}
1218
Perec2922f2016-01-27 15:25:46 +01001219MediaCodecVideoEncoderFactory::~MediaCodecVideoEncoderFactory() {
1220 ALOGD << "MediaCodecVideoEncoderFactory dtor";
perkj461121c2016-02-15 06:28:36 -08001221 if (egl_context_) {
1222 JNIEnv* jni = AttachCurrentThreadIfNeeded();
1223 jni->DeleteGlobalRef(egl_context_);
1224 }
Perec2922f2016-01-27 15:25:46 +01001225}
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001226
perkj30e91822015-11-20 01:31:25 -08001227void MediaCodecVideoEncoderFactory::SetEGLContext(
perkj461121c2016-02-15 06:28:36 -08001228 JNIEnv* jni, jobject egl_context) {
perkj30e91822015-11-20 01:31:25 -08001229 ALOGD << "MediaCodecVideoEncoderFactory::SetEGLContext";
Perfd22e6c2016-02-18 11:35:48 +01001230 if (egl_context_) {
1231 jni->DeleteGlobalRef(egl_context_);
1232 egl_context_ = nullptr;
1233 }
perkj461121c2016-02-15 06:28:36 -08001234 egl_context_ = jni->NewGlobalRef(egl_context);
1235 if (CheckException(jni)) {
1236 ALOGE << "error calling NewGlobalRef for EGL Context.";
perkj30e91822015-11-20 01:31:25 -08001237 }
1238}
1239
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001240webrtc::VideoEncoder* MediaCodecVideoEncoderFactory::CreateVideoEncoder(
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001241 VideoCodecType type) {
1242 if (supported_codecs_.empty()) {
Alex Glaznevad948c42015-11-18 13:06:42 -08001243 ALOGW << "No HW video encoder for type " << (int)type;
Perec2922f2016-01-27 15:25:46 +01001244 return nullptr;
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001245 }
1246 for (std::vector<VideoCodec>::const_iterator it = supported_codecs_.begin();
1247 it != supported_codecs_.end(); ++it) {
1248 if (it->type == type) {
Alex Glaznevfddf6e52015-10-07 16:51:02 -07001249 ALOGD << "Create HW video encoder for type " << (int)type <<
1250 " (" << it->name << ").";
perkj30e91822015-11-20 01:31:25 -08001251 return new MediaCodecVideoEncoder(AttachCurrentThreadIfNeeded(), type,
perkj461121c2016-02-15 06:28:36 -08001252 egl_context_);
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001253 }
1254 }
Alex Glaznevad948c42015-11-18 13:06:42 -08001255 ALOGW << "Can not find HW video encoder for type " << (int)type;
Perec2922f2016-01-27 15:25:46 +01001256 return nullptr;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001257}
1258
1259const std::vector<MediaCodecVideoEncoderFactory::VideoCodec>&
1260MediaCodecVideoEncoderFactory::codecs() const {
1261 return supported_codecs_;
1262}
1263
1264void MediaCodecVideoEncoderFactory::DestroyVideoEncoder(
1265 webrtc::VideoEncoder* encoder) {
Alex Glaznevfddf6e52015-10-07 16:51:02 -07001266 ALOGD << "Destroy video encoder.";
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001267 delete encoder;
1268}
1269
1270} // namespace webrtc_jni