blob: ce1ebc17c07749a00be658de0f08910f0e606d6c [file] [log] [blame]
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001/*
kjellanderb24317b2016-02-10 07:54:43 -08002 * Copyright 2015 The WebRTC project authors. All Rights Reserved.
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00003 *
kjellanderb24317b2016-02-10 07:54:43 -08004 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00009 */
10
kjellandera96e2d72016-02-04 23:52:28 -080011// NOTICE: androidmediaencoder_jni.h must be included before
12// androidmediacodeccommon.h to avoid build errors.
Henrik Kjellander15583c12016-02-10 10:53:12 +010013#include "webrtc/api/java/jni/androidmediaencoder_jni.h"
14
Perba7dc722016-04-19 15:01:23 +020015#include <algorithm>
kwibergd1fe2812016-04-27 06:47:29 -070016#include <memory>
Perba7dc722016-04-19 15:01:23 +020017#include <list>
18
kjellandera96e2d72016-02-04 23:52:28 -080019#include "third_party/libyuv/include/libyuv/convert.h"
20#include "third_party/libyuv/include/libyuv/convert_from.h"
21#include "third_party/libyuv/include/libyuv/video_common.h"
Henrik Kjellander15583c12016-02-10 10:53:12 +010022#include "webrtc/api/java/jni/androidmediacodeccommon.h"
23#include "webrtc/api/java/jni/classreferenceholder.h"
24#include "webrtc/api/java/jni/native_handle_impl.h"
glaznev@webrtc.org18c92472015-02-18 18:42:55 +000025#include "webrtc/base/bind.h"
26#include "webrtc/base/checks.h"
27#include "webrtc/base/logging.h"
28#include "webrtc/base/thread.h"
perkj9576e542015-11-12 06:43:16 -080029#include "webrtc/base/thread_checker.h"
Niels Möllerd28db7f2016-05-10 16:31:47 +020030#include "webrtc/base/timeutils.h"
asapersson1d61a512016-01-20 01:13:46 -080031#include "webrtc/common_types.h"
perkj30e91822015-11-20 01:31:25 -080032#include "webrtc/modules/video_coding/include/video_codec_interface.h"
Peter Boströmcc1543a2016-05-24 12:16:26 +020033#include "webrtc/modules/video_coding/utility/h264_bitstream_parser.h"
kjellander@webrtc.orgb7ce9642015-11-18 23:04:10 +010034#include "webrtc/modules/video_coding/utility/quality_scaler.h"
35#include "webrtc/modules/video_coding/utility/vp8_header_parser.h"
Henrik Kjellander98f53512015-10-28 18:17:40 +010036#include "webrtc/system_wrappers/include/field_trial.h"
37#include "webrtc/system_wrappers/include/logcat_trace_context.h"
glaznev@webrtc.org18c92472015-02-18 18:42:55 +000038
39using rtc::Bind;
40using rtc::Thread;
41using rtc::ThreadManager;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +000042
43using webrtc::CodecSpecificInfo;
44using webrtc::EncodedImage;
Miguel Casas-Sanchez47650702015-05-29 17:21:40 -070045using webrtc::VideoFrame;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +000046using webrtc::RTPFragmentationHeader;
47using webrtc::VideoCodec;
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +000048using webrtc::VideoCodecType;
49using webrtc::kVideoCodecH264;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +000050using webrtc::kVideoCodecVP8;
Alex Glaznevad948c42015-11-18 13:06:42 -080051using webrtc::kVideoCodecVP9;
Alex Glazneva9d08922016-02-19 15:24:06 -080052using webrtc::QualityScaler;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +000053
54namespace webrtc_jni {
55
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +000056// H.264 start code length.
57#define H264_SC_LENGTH 4
58// Maximum allowed NALUs in one output frame.
59#define MAX_NALUS_PERFRAME 32
60// Maximum supported HW video encoder resolution.
61#define MAX_VIDEO_WIDTH 1280
62#define MAX_VIDEO_HEIGHT 1280
63// Maximum supported HW video encoder fps.
64#define MAX_VIDEO_FPS 30
glaznevf4decb52016-01-15 13:49:22 -080065// Maximum allowed fps value in SetRates() call.
66#define MAX_ALLOWED_VIDEO_FPS 60
67// Maximum allowed frames in encoder input queue.
68#define MAX_ENCODER_Q_SIZE 2
glaznev919ff752016-01-27 15:01:03 -080069// Maximum amount of dropped frames caused by full encoder queue - exceeding
70// this threshold means that encoder probably got stuck and need to be reset.
71#define ENCODER_STALL_FRAMEDROP_THRESHOLD 60
glaznevf4decb52016-01-15 13:49:22 -080072
73// Logging macros.
74#define TAG_ENCODER "MediaCodecVideoEncoder"
75#ifdef TRACK_BUFFER_TIMING
76#define ALOGV(...)
77 __android_log_print(ANDROID_LOG_VERBOSE, TAG_ENCODER, __VA_ARGS__)
78#else
79#define ALOGV(...)
80#endif
81#define ALOGD LOG_TAG(rtc::LS_INFO, TAG_ENCODER)
82#define ALOGW LOG_TAG(rtc::LS_WARNING, TAG_ENCODER)
83#define ALOGE LOG_TAG(rtc::LS_ERROR, TAG_ENCODER)
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +000084
asapersson1d61a512016-01-20 01:13:46 -080085namespace {
86// Maximum time limit between incoming frames before requesting a key frame.
87const size_t kFrameDiffThresholdMs = 1100;
88const int kMinKeyFrameInterval = 2;
89} // namespace
90
glaznev@webrtc.org18c92472015-02-18 18:42:55 +000091// MediaCodecVideoEncoder is a webrtc::VideoEncoder implementation that uses
92// Android's MediaCodec SDK API behind the scenes to implement (hopefully)
93// HW-backed video encode. This C++ class is implemented as a very thin shim,
94// delegating all of the interesting work to org.webrtc.MediaCodecVideoEncoder.
95// MediaCodecVideoEncoder is created, operated, and destroyed on a single
96// thread, currently the libjingle Worker thread.
97class MediaCodecVideoEncoder : public webrtc::VideoEncoder,
98 public rtc::MessageHandler {
99 public:
100 virtual ~MediaCodecVideoEncoder();
perkj9576e542015-11-12 06:43:16 -0800101 MediaCodecVideoEncoder(JNIEnv* jni,
perkj30e91822015-11-20 01:31:25 -0800102 VideoCodecType codecType,
103 jobject egl_context);
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000104
105 // webrtc::VideoEncoder implementation. Everything trampolines to
106 // |codec_thread_| for execution.
107 int32_t InitEncode(const webrtc::VideoCodec* codec_settings,
108 int32_t /* number_of_cores */,
109 size_t /* max_payload_size */) override;
pbos22993e12015-10-19 02:39:06 -0700110 int32_t Encode(const webrtc::VideoFrame& input_image,
111 const webrtc::CodecSpecificInfo* /* codec_specific_info */,
112 const std::vector<webrtc::FrameType>* frame_types) override;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000113 int32_t RegisterEncodeCompleteCallback(
114 webrtc::EncodedImageCallback* callback) override;
115 int32_t Release() override;
116 int32_t SetChannelParameters(uint32_t /* packet_loss */,
117 int64_t /* rtt */) override;
118 int32_t SetRates(uint32_t new_bit_rate, uint32_t frame_rate) override;
119
120 // rtc::MessageHandler implementation.
121 void OnMessage(rtc::Message* msg) override;
122
jackychen61b4d512015-04-21 15:30:11 -0700123 void OnDroppedFrame() override;
124
Perec2922f2016-01-27 15:25:46 +0100125 bool SupportsNativeHandle() const override { return egl_context_ != nullptr; }
Peter Boströmb7d9a972015-12-18 16:01:11 +0100126 const char* ImplementationName() const override;
127
128 private:
perkj9576e542015-11-12 06:43:16 -0800129 // ResetCodecOnCodecThread() calls ReleaseOnCodecThread() and
130 // InitEncodeOnCodecThread() in an attempt to restore the codec to an
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000131 // operable state. Necessary after all manner of OMX-layer errors.
perkj9576e542015-11-12 06:43:16 -0800132 bool ResetCodecOnCodecThread();
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000133
134 // Implementation of webrtc::VideoEncoder methods above, all running on the
135 // codec thread exclusively.
136 //
137 // If width==0 then this is assumed to be a re-initialization and the
138 // previously-current values are reused instead of the passed parameters
139 // (makes it easier to reason about thread-safety).
perkj30e91822015-11-20 01:31:25 -0800140 int32_t InitEncodeOnCodecThread(int width, int height, int kbps, int fps,
141 bool use_surface);
142 // Reconfigure to match |frame| in width, height. Also reconfigures the
143 // encoder if |frame| is a texture/byte buffer and the encoder is initialized
144 // for byte buffer/texture. Returns false if reconfiguring fails.
perkj9576e542015-11-12 06:43:16 -0800145 bool MaybeReconfigureEncoderOnCodecThread(const webrtc::VideoFrame& frame);
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000146 int32_t EncodeOnCodecThread(
Miguel Casas-Sanchez47650702015-05-29 17:21:40 -0700147 const webrtc::VideoFrame& input_image,
pbos22993e12015-10-19 02:39:06 -0700148 const std::vector<webrtc::FrameType>* frame_types);
perkj9576e542015-11-12 06:43:16 -0800149 bool EncodeByteBufferOnCodecThread(JNIEnv* jni,
150 bool key_frame, const webrtc::VideoFrame& frame, int input_buffer_index);
perkj30e91822015-11-20 01:31:25 -0800151 bool EncodeTextureOnCodecThread(JNIEnv* jni,
152 bool key_frame, const webrtc::VideoFrame& frame);
perkj9576e542015-11-12 06:43:16 -0800153
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000154 int32_t RegisterEncodeCompleteCallbackOnCodecThread(
155 webrtc::EncodedImageCallback* callback);
156 int32_t ReleaseOnCodecThread();
157 int32_t SetRatesOnCodecThread(uint32_t new_bit_rate, uint32_t frame_rate);
Peter Boström53edac62016-04-27 00:08:34 +0200158 void OnDroppedFrameOnCodecThread();
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000159
160 // Helper accessors for MediaCodecVideoEncoder$OutputBufferInfo members.
161 int GetOutputBufferInfoIndex(JNIEnv* jni, jobject j_output_buffer_info);
162 jobject GetOutputBufferInfoBuffer(JNIEnv* jni, jobject j_output_buffer_info);
163 bool GetOutputBufferInfoIsKeyFrame(JNIEnv* jni, jobject j_output_buffer_info);
164 jlong GetOutputBufferInfoPresentationTimestampUs(
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000165 JNIEnv* jni, jobject j_output_buffer_info);
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000166
167 // Deliver any outputs pending in the MediaCodec to our |callback_| and return
168 // true on success.
169 bool DeliverPendingOutputs(JNIEnv* jni);
170
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000171 // Search for H.264 start codes.
172 int32_t NextNaluPosition(uint8_t *buffer, size_t buffer_size);
173
glaznev94291482016-02-01 13:17:18 -0800174 // Displays encoder statistics.
175 void LogStatistics(bool force_log);
176
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000177 // Type of video codec.
178 VideoCodecType codecType_;
179
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000180 // Valid all the time since RegisterEncodeCompleteCallback() Invoke()s to
181 // |codec_thread_| synchronously.
182 webrtc::EncodedImageCallback* callback_;
183
184 // State that is constant for the lifetime of this object once the ctor
185 // returns.
kwibergd1fe2812016-04-27 06:47:29 -0700186 std::unique_ptr<Thread>
187 codec_thread_; // Thread on which to operate MediaCodec.
perkj9576e542015-11-12 06:43:16 -0800188 rtc::ThreadChecker codec_thread_checker_;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000189 ScopedGlobalRef<jclass> j_media_codec_video_encoder_class_;
190 ScopedGlobalRef<jobject> j_media_codec_video_encoder_;
191 jmethodID j_init_encode_method_;
perkj9576e542015-11-12 06:43:16 -0800192 jmethodID j_get_input_buffers_method_;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000193 jmethodID j_dequeue_input_buffer_method_;
perkj9576e542015-11-12 06:43:16 -0800194 jmethodID j_encode_buffer_method_;
perkj30e91822015-11-20 01:31:25 -0800195 jmethodID j_encode_texture_method_;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000196 jmethodID j_release_method_;
197 jmethodID j_set_rates_method_;
198 jmethodID j_dequeue_output_buffer_method_;
199 jmethodID j_release_output_buffer_method_;
200 jfieldID j_color_format_field_;
201 jfieldID j_info_index_field_;
202 jfieldID j_info_buffer_field_;
203 jfieldID j_info_is_key_frame_field_;
204 jfieldID j_info_presentation_timestamp_us_field_;
205
206 // State that is valid only between InitEncode() and the next Release().
207 // Touched only on codec_thread_ so no explicit synchronization necessary.
208 int width_; // Frame width in pixels.
209 int height_; // Frame height in pixels.
210 bool inited_;
perkj30e91822015-11-20 01:31:25 -0800211 bool use_surface_;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000212 uint16_t picture_id_;
213 enum libyuv::FourCC encoder_fourcc_; // Encoder color space format.
214 int last_set_bitrate_kbps_; // Last-requested bitrate in kbps.
215 int last_set_fps_; // Last-requested frame rate.
216 int64_t current_timestamp_us_; // Current frame timestamps in us.
217 int frames_received_; // Number of frames received by encoder.
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000218 int frames_encoded_; // Number of frames encoded by encoder.
glaznev919ff752016-01-27 15:01:03 -0800219 int frames_dropped_media_encoder_; // Number of frames dropped by encoder.
220 // Number of dropped frames caused by full queue.
221 int consecutive_full_queue_frame_drops_;
glaznev94291482016-02-01 13:17:18 -0800222 int64_t stat_start_time_ms_; // Start time for statistics.
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000223 int current_frames_; // Number of frames in the current statistics interval.
224 int current_bytes_; // Encoded bytes in the current statistics interval.
glaznevf4decb52016-01-15 13:49:22 -0800225 int current_acc_qp_; // Accumulated QP in the current statistics interval.
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000226 int current_encoding_time_ms_; // Overall encoding time in the current second
227 int64_t last_input_timestamp_ms_; // Timestamp of last received yuv frame.
228 int64_t last_output_timestamp_ms_; // Timestamp of last encoded frame.
Perba7dc722016-04-19 15:01:23 +0200229
230 struct InputFrameInfo {
231 InputFrameInfo(int64_t encode_start_time,
232 int32_t frame_timestamp,
233 int64_t frame_render_time_ms,
234 webrtc::VideoRotation rotation)
235 : encode_start_time(encode_start_time),
236 frame_timestamp(frame_timestamp),
237 frame_render_time_ms(frame_render_time_ms),
238 rotation(rotation) {}
239 // Time when video frame is sent to encoder input.
240 const int64_t encode_start_time;
241
242 // Input frame information.
243 const int32_t frame_timestamp;
244 const int64_t frame_render_time_ms;
245 const webrtc::VideoRotation rotation;
246 };
247 std::list<InputFrameInfo> input_frame_infos_;
248 int32_t output_timestamp_; // Last output frame timestamp from
249 // |input_frame_infos_|.
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000250 int64_t output_render_time_ms_; // Last output frame render time from
Perba7dc722016-04-19 15:01:23 +0200251 // |input_frame_infos_|.
252 webrtc::VideoRotation output_rotation_; // Last output frame rotation from
253 // |input_frame_infos_|.
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000254 // Frame size in bytes fed to MediaCodec.
255 int yuv_size_;
256 // True only when between a callback_->Encoded() call return a positive value
257 // and the next Encode() call being ignored.
258 bool drop_next_input_frame_;
259 // Global references; must be deleted in Release().
260 std::vector<jobject> input_buffers_;
Alex Glazneva9d08922016-02-19 15:24:06 -0800261 QualityScaler quality_scaler_;
jackychen61b4d512015-04-21 15:30:11 -0700262 // Dynamic resolution change, off by default.
263 bool scale_;
Peter Boström2bc68c72015-09-24 16:22:28 +0200264
265 // H264 bitstream parser, used to extract QP from encoded bitstreams.
266 webrtc::H264BitstreamParser h264_bitstream_parser_;
Alex Glaznevad948c42015-11-18 13:06:42 -0800267
268 // VP9 variables to populate codec specific structure.
269 webrtc::GofInfoVP9 gof_; // Contains each frame's temporal information for
270 // non-flexible VP9 mode.
271 uint8_t tl0_pic_idx_;
272 size_t gof_idx_;
perkj30e91822015-11-20 01:31:25 -0800273
274 // EGL context - owned by factory, should not be allocated/destroyed
275 // by MediaCodecVideoEncoder.
276 jobject egl_context_;
asapersson1d61a512016-01-20 01:13:46 -0800277
278 // Temporary fix for VP8.
279 // Sends a key frame if frames are largely spaced apart (possibly
280 // corresponding to a large image change).
281 int64_t last_frame_received_ms_;
282 int frames_received_since_last_key_;
283 webrtc::VideoCodecMode codec_mode_;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000284};
285
286MediaCodecVideoEncoder::~MediaCodecVideoEncoder() {
287 // Call Release() to ensure no more callbacks to us after we are deleted.
288 Release();
289}
290
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000291MediaCodecVideoEncoder::MediaCodecVideoEncoder(
perkj30e91822015-11-20 01:31:25 -0800292 JNIEnv* jni, VideoCodecType codecType, jobject egl_context) :
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000293 codecType_(codecType),
294 callback_(NULL),
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000295 codec_thread_(new Thread()),
296 j_media_codec_video_encoder_class_(
297 jni,
298 FindClass(jni, "org/webrtc/MediaCodecVideoEncoder")),
299 j_media_codec_video_encoder_(
300 jni,
301 jni->NewObject(*j_media_codec_video_encoder_class_,
302 GetMethodID(jni,
303 *j_media_codec_video_encoder_class_,
304 "<init>",
perkj30e91822015-11-20 01:31:25 -0800305 "()V"))),
kjellander60ca31b2016-01-04 10:15:53 -0800306 inited_(false),
307 use_surface_(false),
308 picture_id_(0),
perkj30e91822015-11-20 01:31:25 -0800309 egl_context_(egl_context) {
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000310 ScopedLocalRefFrame local_ref_frame(jni);
311 // It would be nice to avoid spinning up a new thread per MediaCodec, and
312 // instead re-use e.g. the PeerConnectionFactory's |worker_thread_|, but bug
313 // 2732 means that deadlocks abound. This class synchronously trampolines
314 // to |codec_thread_|, so if anything else can be coming to _us_ from
315 // |codec_thread_|, or from any thread holding the |_sendCritSect| described
316 // in the bug, we have a problem. For now work around that with a dedicated
317 // thread.
318 codec_thread_->SetName("MediaCodecVideoEncoder", NULL);
henrikg91d6ede2015-09-17 00:24:34 -0700319 RTC_CHECK(codec_thread_->Start()) << "Failed to start MediaCodecVideoEncoder";
perkj9576e542015-11-12 06:43:16 -0800320 codec_thread_checker_.DetachFromThread();
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000321 jclass j_output_buffer_info_class =
322 FindClass(jni, "org/webrtc/MediaCodecVideoEncoder$OutputBufferInfo");
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000323 j_init_encode_method_ = GetMethodID(
324 jni,
325 *j_media_codec_video_encoder_class_,
326 "initEncode",
perkj30e91822015-11-20 01:31:25 -0800327 "(Lorg/webrtc/MediaCodecVideoEncoder$VideoCodecType;"
perkj48477c12015-12-18 00:34:37 -0800328 "IIIILorg/webrtc/EglBase14$Context;)Z");
perkj9576e542015-11-12 06:43:16 -0800329 j_get_input_buffers_method_ = GetMethodID(
330 jni,
331 *j_media_codec_video_encoder_class_,
332 "getInputBuffers",
333 "()[Ljava/nio/ByteBuffer;");
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000334 j_dequeue_input_buffer_method_ = GetMethodID(
335 jni, *j_media_codec_video_encoder_class_, "dequeueInputBuffer", "()I");
perkj9576e542015-11-12 06:43:16 -0800336 j_encode_buffer_method_ = GetMethodID(
337 jni, *j_media_codec_video_encoder_class_, "encodeBuffer", "(ZIIJ)Z");
perkj30e91822015-11-20 01:31:25 -0800338 j_encode_texture_method_ = GetMethodID(
339 jni, *j_media_codec_video_encoder_class_, "encodeTexture",
340 "(ZI[FJ)Z");
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000341 j_release_method_ =
342 GetMethodID(jni, *j_media_codec_video_encoder_class_, "release", "()V");
343 j_set_rates_method_ = GetMethodID(
344 jni, *j_media_codec_video_encoder_class_, "setRates", "(II)Z");
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000345 j_dequeue_output_buffer_method_ = GetMethodID(
346 jni,
347 *j_media_codec_video_encoder_class_,
348 "dequeueOutputBuffer",
349 "()Lorg/webrtc/MediaCodecVideoEncoder$OutputBufferInfo;");
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000350 j_release_output_buffer_method_ = GetMethodID(
351 jni, *j_media_codec_video_encoder_class_, "releaseOutputBuffer", "(I)Z");
352
353 j_color_format_field_ =
354 GetFieldID(jni, *j_media_codec_video_encoder_class_, "colorFormat", "I");
355 j_info_index_field_ =
356 GetFieldID(jni, j_output_buffer_info_class, "index", "I");
357 j_info_buffer_field_ = GetFieldID(
358 jni, j_output_buffer_info_class, "buffer", "Ljava/nio/ByteBuffer;");
359 j_info_is_key_frame_field_ =
360 GetFieldID(jni, j_output_buffer_info_class, "isKeyFrame", "Z");
361 j_info_presentation_timestamp_us_field_ = GetFieldID(
362 jni, j_output_buffer_info_class, "presentationTimestampUs", "J");
363 CHECK_EXCEPTION(jni) << "MediaCodecVideoEncoder ctor failed";
Alex Glaznevad948c42015-11-18 13:06:42 -0800364 srand(time(NULL));
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000365 AllowBlockingCalls();
366}
367
368int32_t MediaCodecVideoEncoder::InitEncode(
369 const webrtc::VideoCodec* codec_settings,
370 int32_t /* number_of_cores */,
371 size_t /* max_payload_size */) {
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000372 if (codec_settings == NULL) {
Alex Glaznevfddf6e52015-10-07 16:51:02 -0700373 ALOGE << "NULL VideoCodec instance";
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000374 return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
375 }
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000376 // Factory should guard against other codecs being used with us.
henrikg91d6ede2015-09-17 00:24:34 -0700377 RTC_CHECK(codec_settings->codecType == codecType_)
378 << "Unsupported codec " << codec_settings->codecType << " for "
379 << codecType_;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000380
asapersson1d61a512016-01-20 01:13:46 -0800381 codec_mode_ = codec_settings->mode;
Alex Glazneva9d08922016-02-19 15:24:06 -0800382 int init_width = codec_settings->width;
383 int init_height = codec_settings->height;
Peter Boström7ace4882016-04-14 00:54:56 +0200384 scale_ = codecType_ != kVideoCodecVP9;
Alex Glazneva9d08922016-02-19 15:24:06 -0800385
386 ALOGD << "InitEncode request: " << init_width << " x " << init_height;
Alex Glaznevfddf6e52015-10-07 16:51:02 -0700387 ALOGD << "Encoder automatic resize " << (scale_ ? "enabled" : "disabled");
Alex Glazneva9d08922016-02-19 15:24:06 -0800388
Peter Boström2bc68c72015-09-24 16:22:28 +0200389 if (scale_) {
390 if (codecType_ == kVideoCodecVP8) {
pbos1f534522016-05-13 11:05:35 -0700391 quality_scaler_.Init(
392 QualityScaler::kLowVp8QpThreshold, QualityScaler::kBadVp8QpThreshold,
393 codec_settings->startBitrate, codec_settings->width,
394 codec_settings->height, codec_settings->maxFramerate);
Peter Boström2bc68c72015-09-24 16:22:28 +0200395 } else if (codecType_ == kVideoCodecH264) {
pbos1f534522016-05-13 11:05:35 -0700396 quality_scaler_.Init(QualityScaler::kLowH264QpThreshold,
397 QualityScaler::kBadH264QpThreshold,
pboscbac40d2016-04-13 02:51:02 -0700398 codec_settings->startBitrate, codec_settings->width,
399 codec_settings->height,
400 codec_settings->maxFramerate);
Peter Boström2bc68c72015-09-24 16:22:28 +0200401 } else {
402 // When adding codec support to additional hardware codecs, also configure
403 // their QP thresholds for scaling.
404 RTC_NOTREACHED() << "Unsupported codec without configured QP thresholds.";
Alex Glazneva9d08922016-02-19 15:24:06 -0800405 scale_ = false;
Peter Boström2bc68c72015-09-24 16:22:28 +0200406 }
Alex Glazneva9d08922016-02-19 15:24:06 -0800407 QualityScaler::Resolution res = quality_scaler_.GetScaledResolution();
Peter Boström926dfcd2016-04-14 14:48:10 +0200408 init_width = res.width;
409 init_height = res.height;
Alex Glazneva9d08922016-02-19 15:24:06 -0800410 ALOGD << "Scaled resolution: " << init_width << " x " << init_height;
jackychen61b4d512015-04-21 15:30:11 -0700411 }
Alex Glazneva9d08922016-02-19 15:24:06 -0800412
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000413 return codec_thread_->Invoke<int32_t>(
Taylor Brandstetter5d97a9a2016-06-10 14:17:27 -0700414 RTC_FROM_HERE,
415 Bind(&MediaCodecVideoEncoder::InitEncodeOnCodecThread, this, init_width,
416 init_height, codec_settings->startBitrate,
skvlad3abb7642016-06-16 12:08:03 -0700417 codec_settings->maxFramerate,
418 codec_settings->expect_encode_from_texture));
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000419}
420
421int32_t MediaCodecVideoEncoder::Encode(
Miguel Casas-Sanchez47650702015-05-29 17:21:40 -0700422 const webrtc::VideoFrame& frame,
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000423 const webrtc::CodecSpecificInfo* /* codec_specific_info */,
pbos22993e12015-10-19 02:39:06 -0700424 const std::vector<webrtc::FrameType>* frame_types) {
Taylor Brandstetter5d97a9a2016-06-10 14:17:27 -0700425 return codec_thread_->Invoke<int32_t>(
426 RTC_FROM_HERE, Bind(&MediaCodecVideoEncoder::EncodeOnCodecThread, this,
427 frame, frame_types));
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000428}
429
430int32_t MediaCodecVideoEncoder::RegisterEncodeCompleteCallback(
431 webrtc::EncodedImageCallback* callback) {
432 return codec_thread_->Invoke<int32_t>(
Taylor Brandstetter5d97a9a2016-06-10 14:17:27 -0700433 RTC_FROM_HERE,
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000434 Bind(&MediaCodecVideoEncoder::RegisterEncodeCompleteCallbackOnCodecThread,
Taylor Brandstetter5d97a9a2016-06-10 14:17:27 -0700435 this, callback));
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000436}
437
438int32_t MediaCodecVideoEncoder::Release() {
Alex Glaznevfddf6e52015-10-07 16:51:02 -0700439 ALOGD << "EncoderRelease request";
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000440 return codec_thread_->Invoke<int32_t>(
Taylor Brandstetter5d97a9a2016-06-10 14:17:27 -0700441 RTC_FROM_HERE, Bind(&MediaCodecVideoEncoder::ReleaseOnCodecThread, this));
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000442}
443
444int32_t MediaCodecVideoEncoder::SetChannelParameters(uint32_t /* packet_loss */,
445 int64_t /* rtt */) {
446 return WEBRTC_VIDEO_CODEC_OK;
447}
448
449int32_t MediaCodecVideoEncoder::SetRates(uint32_t new_bit_rate,
450 uint32_t frame_rate) {
451 return codec_thread_->Invoke<int32_t>(
Taylor Brandstetter5d97a9a2016-06-10 14:17:27 -0700452 RTC_FROM_HERE, Bind(&MediaCodecVideoEncoder::SetRatesOnCodecThread, this,
453 new_bit_rate, frame_rate));
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000454}
455
456void MediaCodecVideoEncoder::OnMessage(rtc::Message* msg) {
perkj9576e542015-11-12 06:43:16 -0800457 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000458 JNIEnv* jni = AttachCurrentThreadIfNeeded();
459 ScopedLocalRefFrame local_ref_frame(jni);
460
461 // We only ever send one message to |this| directly (not through a Bind()'d
462 // functor), so expect no ID/data.
henrikg91d6ede2015-09-17 00:24:34 -0700463 RTC_CHECK(!msg->message_id) << "Unexpected message!";
464 RTC_CHECK(!msg->pdata) << "Unexpected message!";
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000465 if (!inited_) {
466 return;
467 }
468
469 // It would be nice to recover from a failure here if one happened, but it's
470 // unclear how to signal such a failure to the app, so instead we stay silent
471 // about it and let the next app-called API method reveal the borkedness.
472 DeliverPendingOutputs(jni);
sakal47456802016-05-24 00:16:58 -0700473
sakal1fc48102016-06-14 01:53:39 -0700474 // If there aren't more frames to deliver, we can start polling at lower rate.
475 if (input_frame_infos_.empty()) {
476 codec_thread_->PostDelayed(RTC_FROM_HERE, kMediaCodecPollNoFramesMs, this);
sakal47456802016-05-24 00:16:58 -0700477 } else {
sakal1fc48102016-06-14 01:53:39 -0700478 codec_thread_->PostDelayed(RTC_FROM_HERE, kMediaCodecPollMs, this);
sakal47456802016-05-24 00:16:58 -0700479 }
sakal1fc48102016-06-14 01:53:39 -0700480
481 // Call log statistics here so it's called even if no frames are being
482 // delivered.
483 LogStatistics(false);
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000484}
485
perkj9576e542015-11-12 06:43:16 -0800486bool MediaCodecVideoEncoder::ResetCodecOnCodecThread() {
487 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
488 ALOGE << "ResetOnCodecThread";
489 if (ReleaseOnCodecThread() != WEBRTC_VIDEO_CODEC_OK ||
perkj30e91822015-11-20 01:31:25 -0800490 InitEncodeOnCodecThread(width_, height_, 0, 0, false) !=
491 WEBRTC_VIDEO_CODEC_OK) {
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000492 // TODO(fischman): wouldn't it be nice if there was a way to gracefully
493 // degrade to a SW encoder at this point? There isn't one AFAICT :(
494 // https://code.google.com/p/webrtc/issues/detail?id=2920
perkj9576e542015-11-12 06:43:16 -0800495 return false;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000496 }
perkj9576e542015-11-12 06:43:16 -0800497 return true;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000498}
499
500int32_t MediaCodecVideoEncoder::InitEncodeOnCodecThread(
perkj30e91822015-11-20 01:31:25 -0800501 int width, int height, int kbps, int fps, bool use_surface) {
perkj9576e542015-11-12 06:43:16 -0800502 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
perkj30e91822015-11-20 01:31:25 -0800503 RTC_CHECK(!use_surface || egl_context_ != nullptr) << "EGL context not set.";
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000504 JNIEnv* jni = AttachCurrentThreadIfNeeded();
505 ScopedLocalRefFrame local_ref_frame(jni);
506
Alex Glaznevfddf6e52015-10-07 16:51:02 -0700507 ALOGD << "InitEncodeOnCodecThread Type: " << (int)codecType_ << ", " <<
508 width << " x " << height << ". Bitrate: " << kbps <<
509 " kbps. Fps: " << fps;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000510 if (kbps == 0) {
511 kbps = last_set_bitrate_kbps_;
512 }
513 if (fps == 0) {
glaznevf4decb52016-01-15 13:49:22 -0800514 fps = MAX_VIDEO_FPS;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000515 }
516
517 width_ = width;
518 height_ = height;
519 last_set_bitrate_kbps_ = kbps;
glaznevf4decb52016-01-15 13:49:22 -0800520 last_set_fps_ = (fps < MAX_VIDEO_FPS) ? fps : MAX_VIDEO_FPS;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000521 yuv_size_ = width_ * height_ * 3 / 2;
522 frames_received_ = 0;
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000523 frames_encoded_ = 0;
glaznev919ff752016-01-27 15:01:03 -0800524 frames_dropped_media_encoder_ = 0;
525 consecutive_full_queue_frame_drops_ = 0;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000526 current_timestamp_us_ = 0;
Niels Möllerd28db7f2016-05-10 16:31:47 +0200527 stat_start_time_ms_ = rtc::TimeMillis();
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000528 current_frames_ = 0;
529 current_bytes_ = 0;
glaznevf4decb52016-01-15 13:49:22 -0800530 current_acc_qp_ = 0;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000531 current_encoding_time_ms_ = 0;
532 last_input_timestamp_ms_ = -1;
533 last_output_timestamp_ms_ = -1;
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000534 output_timestamp_ = 0;
535 output_render_time_ms_ = 0;
Perba7dc722016-04-19 15:01:23 +0200536 input_frame_infos_.clear();
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000537 drop_next_input_frame_ = false;
perkj30e91822015-11-20 01:31:25 -0800538 use_surface_ = use_surface;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000539 picture_id_ = static_cast<uint16_t>(rand()) & 0x7FFF;
Alex Glaznevad948c42015-11-18 13:06:42 -0800540 gof_.SetGofInfoVP9(webrtc::TemporalStructureMode::kTemporalStructureMode1);
541 tl0_pic_idx_ = static_cast<uint8_t>(rand());
542 gof_idx_ = 0;
asapersson1d61a512016-01-20 01:13:46 -0800543 last_frame_received_ms_ = -1;
544 frames_received_since_last_key_ = kMinKeyFrameInterval;
perkj9576e542015-11-12 06:43:16 -0800545
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000546 // We enforce no extra stride/padding in the format creation step.
Perec2922f2016-01-27 15:25:46 +0100547 jobject j_video_codec_enum = JavaEnumFromIndexAndClassName(
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000548 jni, "MediaCodecVideoEncoder$VideoCodecType", codecType_);
perkj9576e542015-11-12 06:43:16 -0800549 const bool encode_status = jni->CallBooleanMethod(
550 *j_media_codec_video_encoder_, j_init_encode_method_,
perkj30e91822015-11-20 01:31:25 -0800551 j_video_codec_enum, width, height, kbps, fps,
552 (use_surface ? egl_context_ : nullptr));
perkj9576e542015-11-12 06:43:16 -0800553 if (!encode_status) {
554 ALOGE << "Failed to configure encoder.";
555 return WEBRTC_VIDEO_CODEC_ERROR;
556 }
557 CHECK_EXCEPTION(jni);
558
Per598242a2015-11-26 14:28:55 +0100559 if (!use_surface) {
perkj30e91822015-11-20 01:31:25 -0800560 jobjectArray input_buffers = reinterpret_cast<jobjectArray>(
561 jni->CallObjectMethod(*j_media_codec_video_encoder_,
562 j_get_input_buffers_method_));
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000563 CHECK_EXCEPTION(jni);
perkj30e91822015-11-20 01:31:25 -0800564 if (IsNull(jni, input_buffers)) {
565 return WEBRTC_VIDEO_CODEC_ERROR;
566 }
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000567
perkj30e91822015-11-20 01:31:25 -0800568 switch (GetIntField(jni, *j_media_codec_video_encoder_,
569 j_color_format_field_)) {
570 case COLOR_FormatYUV420Planar:
571 encoder_fourcc_ = libyuv::FOURCC_YU12;
572 break;
573 case COLOR_FormatYUV420SemiPlanar:
574 case COLOR_QCOM_FormatYUV420SemiPlanar:
575 case COLOR_QCOM_FORMATYUV420PackedSemiPlanar32m:
576 encoder_fourcc_ = libyuv::FOURCC_NV12;
577 break;
578 default:
579 LOG(LS_ERROR) << "Wrong color format.";
580 return WEBRTC_VIDEO_CODEC_ERROR;
581 }
582 size_t num_input_buffers = jni->GetArrayLength(input_buffers);
583 RTC_CHECK(input_buffers_.empty())
584 << "Unexpected double InitEncode without Release";
585 input_buffers_.resize(num_input_buffers);
586 for (size_t i = 0; i < num_input_buffers; ++i) {
587 input_buffers_[i] =
588 jni->NewGlobalRef(jni->GetObjectArrayElement(input_buffers, i));
589 int64_t yuv_buffer_capacity =
590 jni->GetDirectBufferCapacity(input_buffers_[i]);
591 CHECK_EXCEPTION(jni);
592 RTC_CHECK(yuv_buffer_capacity >= yuv_size_) << "Insufficient capacity";
593 }
594 }
perkj9576e542015-11-12 06:43:16 -0800595
596 inited_ = true;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000597 return WEBRTC_VIDEO_CODEC_OK;
598}
599
600int32_t MediaCodecVideoEncoder::EncodeOnCodecThread(
Miguel Casas-Sanchez47650702015-05-29 17:21:40 -0700601 const webrtc::VideoFrame& frame,
pbos22993e12015-10-19 02:39:06 -0700602 const std::vector<webrtc::FrameType>* frame_types) {
perkj9576e542015-11-12 06:43:16 -0800603 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000604 JNIEnv* jni = AttachCurrentThreadIfNeeded();
605 ScopedLocalRefFrame local_ref_frame(jni);
606
607 if (!inited_) {
608 return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
609 }
perkj9576e542015-11-12 06:43:16 -0800610
asapersson1d61a512016-01-20 01:13:46 -0800611 bool send_key_frame = false;
glaznev94291482016-02-01 13:17:18 -0800612 if (codec_mode_ == webrtc::kRealtimeVideo) {
asapersson1d61a512016-01-20 01:13:46 -0800613 ++frames_received_since_last_key_;
Niels Möllerd28db7f2016-05-10 16:31:47 +0200614 int64_t now_ms = rtc::TimeMillis();
asapersson1d61a512016-01-20 01:13:46 -0800615 if (last_frame_received_ms_ != -1 &&
616 (now_ms - last_frame_received_ms_) > kFrameDiffThresholdMs) {
617 // Add limit to prevent triggering a key for every frame for very low
618 // framerates (e.g. if frame diff > kFrameDiffThresholdMs).
619 if (frames_received_since_last_key_ > kMinKeyFrameInterval) {
620 ALOGD << "Send key, frame diff: " << (now_ms - last_frame_received_ms_);
621 send_key_frame = true;
622 }
623 frames_received_since_last_key_ = 0;
624 }
625 last_frame_received_ms_ = now_ms;
626 }
627
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000628 frames_received_++;
629 if (!DeliverPendingOutputs(jni)) {
perkj9576e542015-11-12 06:43:16 -0800630 if (!ResetCodecOnCodecThread())
631 return WEBRTC_VIDEO_CODEC_ERROR;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000632 }
glaznevf4decb52016-01-15 13:49:22 -0800633 if (frames_encoded_ < kMaxEncodedLogFrames) {
Perba7dc722016-04-19 15:01:23 +0200634 ALOGD << "Encoder frame in # " << (frames_received_ - 1)
635 << ". TS: " << (int)(current_timestamp_us_ / 1000)
636 << ". Q: " << input_frame_infos_.size() << ". Fps: " << last_set_fps_
637 << ". Kbps: " << last_set_bitrate_kbps_;
glaznevf4decb52016-01-15 13:49:22 -0800638 }
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000639
640 if (drop_next_input_frame_) {
perkj9576e542015-11-12 06:43:16 -0800641 ALOGW << "Encoder drop frame - failed callback.";
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000642 drop_next_input_frame_ = false;
glaznevf4decb52016-01-15 13:49:22 -0800643 current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
glaznev919ff752016-01-27 15:01:03 -0800644 frames_dropped_media_encoder_++;
Peter Boström53edac62016-04-27 00:08:34 +0200645 OnDroppedFrameOnCodecThread();
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000646 return WEBRTC_VIDEO_CODEC_OK;
647 }
648
henrikg91d6ede2015-09-17 00:24:34 -0700649 RTC_CHECK(frame_types->size() == 1) << "Unexpected stream count";
Peter Boström2bc68c72015-09-24 16:22:28 +0200650
Peter Boströmf7704d12016-04-11 16:42:40 +0200651 // Check if we accumulated too many frames in encoder input buffers and drop
652 // frame if so.
Perba7dc722016-04-19 15:01:23 +0200653 if (input_frame_infos_.size() > MAX_ENCODER_Q_SIZE) {
654 ALOGD << "Already " << input_frame_infos_.size()
655 << " frames in the queue, dropping"
Peter Boströmf7704d12016-04-11 16:42:40 +0200656 << ". TS: " << (int)(current_timestamp_us_ / 1000)
657 << ". Fps: " << last_set_fps_
658 << ". Consecutive drops: " << consecutive_full_queue_frame_drops_;
659 current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
660 consecutive_full_queue_frame_drops_++;
661 if (consecutive_full_queue_frame_drops_ >=
662 ENCODER_STALL_FRAMEDROP_THRESHOLD) {
663 ALOGE << "Encoder got stuck. Reset.";
664 ResetCodecOnCodecThread();
665 return WEBRTC_VIDEO_CODEC_ERROR;
glaznevf4decb52016-01-15 13:49:22 -0800666 }
Peter Boströmf7704d12016-04-11 16:42:40 +0200667 frames_dropped_media_encoder_++;
Peter Boström53edac62016-04-27 00:08:34 +0200668 OnDroppedFrameOnCodecThread();
Peter Boströmf7704d12016-04-11 16:42:40 +0200669 return WEBRTC_VIDEO_CODEC_OK;
glaznevf4decb52016-01-15 13:49:22 -0800670 }
glaznev919ff752016-01-27 15:01:03 -0800671 consecutive_full_queue_frame_drops_ = 0;
glaznevf4decb52016-01-15 13:49:22 -0800672
nisse72e735d2016-06-17 02:55:14 -0700673 VideoFrame input_frame = frame;
Per598242a2015-11-26 14:28:55 +0100674 if (scale_) {
675 // Check framerate before spatial resolution change.
Niels Möller718a7632016-06-13 13:06:01 +0200676 quality_scaler_.OnEncodeFrame(frame.width(), frame.height());
Per598242a2015-11-26 14:28:55 +0100677 const webrtc::QualityScaler::Resolution scaled_resolution =
678 quality_scaler_.GetScaledResolution();
679 if (scaled_resolution.width != frame.width() ||
680 scaled_resolution.height != frame.height()) {
nisse72e735d2016-06-17 02:55:14 -0700681 if (frame.video_frame_buffer()->native_handle() != nullptr) {
682 rtc::scoped_refptr<webrtc::VideoFrameBuffer> scaled_buffer(
683 static_cast<AndroidTextureBuffer*>(
684 frame.video_frame_buffer().get())->CropScaleAndRotate(
685 frame.width(), frame.height(), 0, 0,
686 scaled_resolution.width, scaled_resolution.height,
687 webrtc::kVideoRotation_0));
688 input_frame.set_video_frame_buffer(scaled_buffer);
Per598242a2015-11-26 14:28:55 +0100689 } else {
nisse72e735d2016-06-17 02:55:14 -0700690 input_frame.set_video_frame_buffer(
691 quality_scaler_.GetScaledBuffer(frame.video_frame_buffer()));
Per598242a2015-11-26 14:28:55 +0100692 }
693 }
694 }
jackychen61b4d512015-04-21 15:30:11 -0700695
perkj9576e542015-11-12 06:43:16 -0800696 if (!MaybeReconfigureEncoderOnCodecThread(input_frame)) {
697 ALOGE << "Failed to reconfigure encoder.";
698 return WEBRTC_VIDEO_CODEC_ERROR;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000699 }
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000700
Niels Möllerd28db7f2016-05-10 16:31:47 +0200701 const int64_t time_before_calling_encode = rtc::TimeMillis();
asapersson1d61a512016-01-20 01:13:46 -0800702 const bool key_frame =
703 frame_types->front() != webrtc::kVideoFrameDelta || send_key_frame;
perkj30e91822015-11-20 01:31:25 -0800704 bool encode_status = true;
nisse26acec42016-04-15 03:43:39 -0700705 if (!input_frame.video_frame_buffer()->native_handle()) {
perkj30e91822015-11-20 01:31:25 -0800706 int j_input_buffer_index = jni->CallIntMethod(*j_media_codec_video_encoder_,
707 j_dequeue_input_buffer_method_);
708 CHECK_EXCEPTION(jni);
709 if (j_input_buffer_index == -1) {
710 // Video codec falls behind - no input buffer available.
711 ALOGW << "Encoder drop frame - no input buffers available";
glaznev919ff752016-01-27 15:01:03 -0800712 current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
713 frames_dropped_media_encoder_++;
Peter Boström53edac62016-04-27 00:08:34 +0200714 OnDroppedFrameOnCodecThread();
perkj30e91822015-11-20 01:31:25 -0800715 return WEBRTC_VIDEO_CODEC_OK; // TODO(fischman): see webrtc bug 2887.
716 }
717 if (j_input_buffer_index == -2) {
718 ResetCodecOnCodecThread();
719 return WEBRTC_VIDEO_CODEC_ERROR;
720 }
721 encode_status = EncodeByteBufferOnCodecThread(jni, key_frame, input_frame,
722 j_input_buffer_index);
723 } else {
724 encode_status = EncodeTextureOnCodecThread(jni, key_frame, input_frame);
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000725 }
perkj30e91822015-11-20 01:31:25 -0800726
727 if (!encode_status) {
728 ALOGE << "Failed encode frame with timestamp: " << input_frame.timestamp();
perkj9576e542015-11-12 06:43:16 -0800729 ResetCodecOnCodecThread();
perkj12f68022015-10-16 13:31:45 +0200730 return WEBRTC_VIDEO_CODEC_ERROR;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000731 }
732
Perba7dc722016-04-19 15:01:23 +0200733 // Save input image timestamps for later output.
734 input_frame_infos_.emplace_back(
735 time_before_calling_encode, input_frame.timestamp(),
736 input_frame.render_time_ms(), input_frame.rotation());
737
perkj9576e542015-11-12 06:43:16 -0800738 last_input_timestamp_ms_ =
739 current_timestamp_us_ / rtc::kNumMicrosecsPerMillisec;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000740
perkj9576e542015-11-12 06:43:16 -0800741 current_timestamp_us_ += rtc::kNumMicrosecsPerSec / last_set_fps_;
742
sakal1fc48102016-06-14 01:53:39 -0700743 codec_thread_->Clear(this);
744 codec_thread_->PostDelayed(RTC_FROM_HERE, kMediaCodecPollMs, this);
sakal47456802016-05-24 00:16:58 -0700745
perkj30e91822015-11-20 01:31:25 -0800746 if (!DeliverPendingOutputs(jni)) {
perkj9576e542015-11-12 06:43:16 -0800747 ALOGE << "Failed deliver pending outputs.";
748 ResetCodecOnCodecThread();
749 return WEBRTC_VIDEO_CODEC_ERROR;
750 }
751 return WEBRTC_VIDEO_CODEC_OK;
752}
753
754bool MediaCodecVideoEncoder::MaybeReconfigureEncoderOnCodecThread(
755 const webrtc::VideoFrame& frame) {
756 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
757
nisse26acec42016-04-15 03:43:39 -0700758 const bool is_texture_frame =
759 frame.video_frame_buffer()->native_handle() != nullptr;
perkj30e91822015-11-20 01:31:25 -0800760 const bool reconfigure_due_to_format = is_texture_frame != use_surface_;
perkj9576e542015-11-12 06:43:16 -0800761 const bool reconfigure_due_to_size =
762 frame.width() != width_ || frame.height() != height_;
763
perkj30e91822015-11-20 01:31:25 -0800764 if (reconfigure_due_to_format) {
765 ALOGD << "Reconfigure encoder due to format change. "
766 << (use_surface_ ?
767 "Reconfiguring to encode from byte buffer." :
768 "Reconfiguring to encode from texture.");
glaznev94291482016-02-01 13:17:18 -0800769 LogStatistics(true);
perkj30e91822015-11-20 01:31:25 -0800770 }
perkj9576e542015-11-12 06:43:16 -0800771 if (reconfigure_due_to_size) {
glaznev94291482016-02-01 13:17:18 -0800772 ALOGW << "Reconfigure encoder due to frame resolution change from "
perkj9576e542015-11-12 06:43:16 -0800773 << width_ << " x " << height_ << " to " << frame.width() << " x "
774 << frame.height();
glaznev94291482016-02-01 13:17:18 -0800775 LogStatistics(true);
perkj9576e542015-11-12 06:43:16 -0800776 width_ = frame.width();
777 height_ = frame.height();
778 }
779
perkj30e91822015-11-20 01:31:25 -0800780 if (!reconfigure_due_to_format && !reconfigure_due_to_size)
perkj9576e542015-11-12 06:43:16 -0800781 return true;
782
783 ReleaseOnCodecThread();
784
perkj30e91822015-11-20 01:31:25 -0800785 return InitEncodeOnCodecThread(width_, height_, 0, 0 , is_texture_frame) ==
perkj9576e542015-11-12 06:43:16 -0800786 WEBRTC_VIDEO_CODEC_OK;
787}
788
789bool MediaCodecVideoEncoder::EncodeByteBufferOnCodecThread(JNIEnv* jni,
790 bool key_frame, const webrtc::VideoFrame& frame, int input_buffer_index) {
791 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
perkj30e91822015-11-20 01:31:25 -0800792 RTC_CHECK(!use_surface_);
perkj9576e542015-11-12 06:43:16 -0800793
perkj9576e542015-11-12 06:43:16 -0800794 jobject j_input_buffer = input_buffers_[input_buffer_index];
795 uint8_t* yuv_buffer =
796 reinterpret_cast<uint8_t*>(jni->GetDirectBufferAddress(j_input_buffer));
797 CHECK_EXCEPTION(jni);
798 RTC_CHECK(yuv_buffer) << "Indirect buffer??";
799 RTC_CHECK(!libyuv::ConvertFromI420(
nissec9c142f2016-05-17 04:05:47 -0700800 frame.video_frame_buffer()->DataY(),
801 frame.video_frame_buffer()->StrideY(),
802 frame.video_frame_buffer()->DataU(),
803 frame.video_frame_buffer()->StrideU(),
804 frame.video_frame_buffer()->DataV(),
805 frame.video_frame_buffer()->StrideV(),
perkj9576e542015-11-12 06:43:16 -0800806 yuv_buffer, width_, width_, height_, encoder_fourcc_))
807 << "ConvertFromI420 failed";
808
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000809 bool encode_status = jni->CallBooleanMethod(*j_media_codec_video_encoder_,
perkj9576e542015-11-12 06:43:16 -0800810 j_encode_buffer_method_,
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000811 key_frame,
perkj9576e542015-11-12 06:43:16 -0800812 input_buffer_index,
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000813 yuv_size_,
814 current_timestamp_us_);
815 CHECK_EXCEPTION(jni);
perkj9576e542015-11-12 06:43:16 -0800816 return encode_status;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000817}
818
perkj30e91822015-11-20 01:31:25 -0800819bool MediaCodecVideoEncoder::EncodeTextureOnCodecThread(JNIEnv* jni,
820 bool key_frame, const webrtc::VideoFrame& frame) {
821 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
822 RTC_CHECK(use_surface_);
nisse26acec42016-04-15 03:43:39 -0700823 NativeHandleImpl* handle = static_cast<NativeHandleImpl*>(
824 frame.video_frame_buffer()->native_handle());
nisse47ac4622016-05-25 08:47:01 -0700825 jfloatArray sampling_matrix = handle->sampling_matrix.ToJava(jni);
perkj30e91822015-11-20 01:31:25 -0800826 bool encode_status = jni->CallBooleanMethod(*j_media_codec_video_encoder_,
827 j_encode_texture_method_,
828 key_frame,
829 handle->oes_texture_id,
830 sampling_matrix,
831 current_timestamp_us_);
832 CHECK_EXCEPTION(jni);
833 return encode_status;
834}
835
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000836int32_t MediaCodecVideoEncoder::RegisterEncodeCompleteCallbackOnCodecThread(
837 webrtc::EncodedImageCallback* callback) {
perkj9576e542015-11-12 06:43:16 -0800838 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000839 JNIEnv* jni = AttachCurrentThreadIfNeeded();
840 ScopedLocalRefFrame local_ref_frame(jni);
841 callback_ = callback;
842 return WEBRTC_VIDEO_CODEC_OK;
843}
844
845int32_t MediaCodecVideoEncoder::ReleaseOnCodecThread() {
perkj9576e542015-11-12 06:43:16 -0800846 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000847 if (!inited_) {
848 return WEBRTC_VIDEO_CODEC_OK;
849 }
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000850 JNIEnv* jni = AttachCurrentThreadIfNeeded();
Alex Glaznevfddf6e52015-10-07 16:51:02 -0700851 ALOGD << "EncoderReleaseOnCodecThread: Frames received: " <<
852 frames_received_ << ". Encoded: " << frames_encoded_ <<
glaznev919ff752016-01-27 15:01:03 -0800853 ". Dropped: " << frames_dropped_media_encoder_;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000854 ScopedLocalRefFrame local_ref_frame(jni);
855 for (size_t i = 0; i < input_buffers_.size(); ++i)
856 jni->DeleteGlobalRef(input_buffers_[i]);
857 input_buffers_.clear();
858 jni->CallVoidMethod(*j_media_codec_video_encoder_, j_release_method_);
859 CHECK_EXCEPTION(jni);
860 rtc::MessageQueueManager::Clear(this);
861 inited_ = false;
perkj30e91822015-11-20 01:31:25 -0800862 use_surface_ = false;
Alex Glaznevfddf6e52015-10-07 16:51:02 -0700863 ALOGD << "EncoderReleaseOnCodecThread done.";
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000864 return WEBRTC_VIDEO_CODEC_OK;
865}
866
867int32_t MediaCodecVideoEncoder::SetRatesOnCodecThread(uint32_t new_bit_rate,
868 uint32_t frame_rate) {
perkj9576e542015-11-12 06:43:16 -0800869 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
glaznevf4decb52016-01-15 13:49:22 -0800870 frame_rate = (frame_rate < MAX_ALLOWED_VIDEO_FPS) ?
871 frame_rate : MAX_ALLOWED_VIDEO_FPS;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000872 if (last_set_bitrate_kbps_ == new_bit_rate &&
873 last_set_fps_ == frame_rate) {
874 return WEBRTC_VIDEO_CODEC_OK;
875 }
glaznev919ff752016-01-27 15:01:03 -0800876 if (scale_) {
877 quality_scaler_.ReportFramerate(frame_rate);
878 }
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000879 JNIEnv* jni = AttachCurrentThreadIfNeeded();
880 ScopedLocalRefFrame local_ref_frame(jni);
881 if (new_bit_rate > 0) {
882 last_set_bitrate_kbps_ = new_bit_rate;
883 }
884 if (frame_rate > 0) {
885 last_set_fps_ = frame_rate;
886 }
887 bool ret = jni->CallBooleanMethod(*j_media_codec_video_encoder_,
888 j_set_rates_method_,
889 last_set_bitrate_kbps_,
890 last_set_fps_);
891 CHECK_EXCEPTION(jni);
892 if (!ret) {
perkj9576e542015-11-12 06:43:16 -0800893 ResetCodecOnCodecThread();
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000894 return WEBRTC_VIDEO_CODEC_ERROR;
895 }
896 return WEBRTC_VIDEO_CODEC_OK;
897}
898
899int MediaCodecVideoEncoder::GetOutputBufferInfoIndex(
900 JNIEnv* jni,
901 jobject j_output_buffer_info) {
902 return GetIntField(jni, j_output_buffer_info, j_info_index_field_);
903}
904
905jobject MediaCodecVideoEncoder::GetOutputBufferInfoBuffer(
906 JNIEnv* jni,
907 jobject j_output_buffer_info) {
908 return GetObjectField(jni, j_output_buffer_info, j_info_buffer_field_);
909}
910
911bool MediaCodecVideoEncoder::GetOutputBufferInfoIsKeyFrame(
912 JNIEnv* jni,
913 jobject j_output_buffer_info) {
914 return GetBooleanField(jni, j_output_buffer_info, j_info_is_key_frame_field_);
915}
916
917jlong MediaCodecVideoEncoder::GetOutputBufferInfoPresentationTimestampUs(
918 JNIEnv* jni,
919 jobject j_output_buffer_info) {
920 return GetLongField(
921 jni, j_output_buffer_info, j_info_presentation_timestamp_us_field_);
922}
923
924bool MediaCodecVideoEncoder::DeliverPendingOutputs(JNIEnv* jni) {
perkj9576e542015-11-12 06:43:16 -0800925 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
sakal1fc48102016-06-14 01:53:39 -0700926
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000927 while (true) {
928 jobject j_output_buffer_info = jni->CallObjectMethod(
929 *j_media_codec_video_encoder_, j_dequeue_output_buffer_method_);
930 CHECK_EXCEPTION(jni);
931 if (IsNull(jni, j_output_buffer_info)) {
932 break;
933 }
934
935 int output_buffer_index =
936 GetOutputBufferInfoIndex(jni, j_output_buffer_info);
937 if (output_buffer_index == -1) {
perkj9576e542015-11-12 06:43:16 -0800938 ResetCodecOnCodecThread();
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000939 return false;
940 }
941
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000942 // Get key and config frame flags.
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000943 jobject j_output_buffer =
944 GetOutputBufferInfoBuffer(jni, j_output_buffer_info);
945 bool key_frame = GetOutputBufferInfoIsKeyFrame(jni, j_output_buffer_info);
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000946
947 // Get frame timestamps from a queue - for non config frames only.
948 int64_t frame_encoding_time_ms = 0;
949 last_output_timestamp_ms_ =
950 GetOutputBufferInfoPresentationTimestampUs(jni, j_output_buffer_info) /
sakal47456802016-05-24 00:16:58 -0700951 rtc::kNumMicrosecsPerMillisec;
Perba7dc722016-04-19 15:01:23 +0200952 if (!input_frame_infos_.empty()) {
953 const InputFrameInfo& frame_info = input_frame_infos_.front();
954 output_timestamp_ = frame_info.frame_timestamp;
955 output_render_time_ms_ = frame_info.frame_render_time_ms;
956 output_rotation_ = frame_info.rotation;
957 frame_encoding_time_ms =
Niels Möllerd28db7f2016-05-10 16:31:47 +0200958 rtc::TimeMillis() - frame_info.encode_start_time;
Perba7dc722016-04-19 15:01:23 +0200959 input_frame_infos_.pop_front();
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000960 }
961
962 // Extract payload.
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000963 size_t payload_size = jni->GetDirectBufferCapacity(j_output_buffer);
Peter Boström0c4e06b2015-10-07 12:23:21 +0200964 uint8_t* payload = reinterpret_cast<uint8_t*>(
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000965 jni->GetDirectBufferAddress(j_output_buffer));
966 CHECK_EXCEPTION(jni);
967
glaznevf4decb52016-01-15 13:49:22 -0800968 if (frames_encoded_ < kMaxEncodedLogFrames) {
glaznev94291482016-02-01 13:17:18 -0800969 int current_latency =
970 (int)(last_input_timestamp_ms_ - last_output_timestamp_ms_);
971 ALOGD << "Encoder frame out # " << frames_encoded_ <<
972 ". Key: " << key_frame <<
973 ". Size: " << payload_size <<
974 ". TS: " << (int)last_output_timestamp_ms_ <<
975 ". Latency: " << current_latency <<
glaznevf4decb52016-01-15 13:49:22 -0800976 ". EncTime: " << frame_encoding_time_ms;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000977 }
978
979 // Callback - return encoded frame.
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000980 int32_t callback_status = 0;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000981 if (callback_) {
kwibergd1fe2812016-04-27 06:47:29 -0700982 std::unique_ptr<webrtc::EncodedImage> image(
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000983 new webrtc::EncodedImage(payload, payload_size, payload_size));
984 image->_encodedWidth = width_;
985 image->_encodedHeight = height_;
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000986 image->_timeStamp = output_timestamp_;
987 image->capture_time_ms_ = output_render_time_ms_;
Perba7dc722016-04-19 15:01:23 +0200988 image->rotation_ = output_rotation_;
Peter Boström49e196a2015-10-23 15:58:18 +0200989 image->_frameType =
990 (key_frame ? webrtc::kVideoFrameKey : webrtc::kVideoFrameDelta);
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000991 image->_completeFrame = true;
asapersson075fb4b2015-10-29 08:49:14 -0700992 image->adapt_reason_.quality_resolution_downscales =
993 scale_ ? quality_scaler_.downscale_shift() : -1;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +0000994
995 webrtc::CodecSpecificInfo info;
996 memset(&info, 0, sizeof(info));
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +0000997 info.codecType = codecType_;
998 if (codecType_ == kVideoCodecVP8) {
999 info.codecSpecific.VP8.pictureId = picture_id_;
1000 info.codecSpecific.VP8.nonReference = false;
1001 info.codecSpecific.VP8.simulcastIdx = 0;
1002 info.codecSpecific.VP8.temporalIdx = webrtc::kNoTemporalIdx;
1003 info.codecSpecific.VP8.layerSync = false;
1004 info.codecSpecific.VP8.tl0PicIdx = webrtc::kNoTl0PicIdx;
1005 info.codecSpecific.VP8.keyIdx = webrtc::kNoKeyIdx;
Alex Glaznevad948c42015-11-18 13:06:42 -08001006 } else if (codecType_ == kVideoCodecVP9) {
1007 if (key_frame) {
1008 gof_idx_ = 0;
1009 }
1010 info.codecSpecific.VP9.picture_id = picture_id_;
1011 info.codecSpecific.VP9.inter_pic_predicted = key_frame ? false : true;
1012 info.codecSpecific.VP9.flexible_mode = false;
1013 info.codecSpecific.VP9.ss_data_available = key_frame ? true : false;
1014 info.codecSpecific.VP9.tl0_pic_idx = tl0_pic_idx_++;
1015 info.codecSpecific.VP9.temporal_idx = webrtc::kNoTemporalIdx;
1016 info.codecSpecific.VP9.spatial_idx = webrtc::kNoSpatialIdx;
1017 info.codecSpecific.VP9.temporal_up_switch = true;
1018 info.codecSpecific.VP9.inter_layer_predicted = false;
1019 info.codecSpecific.VP9.gof_idx =
1020 static_cast<uint8_t>(gof_idx_++ % gof_.num_frames_in_gof);
1021 info.codecSpecific.VP9.num_spatial_layers = 1;
1022 info.codecSpecific.VP9.spatial_layer_resolution_present = false;
1023 if (info.codecSpecific.VP9.ss_data_available) {
1024 info.codecSpecific.VP9.spatial_layer_resolution_present = true;
1025 info.codecSpecific.VP9.width[0] = width_;
1026 info.codecSpecific.VP9.height[0] = height_;
1027 info.codecSpecific.VP9.gof.CopyGofInfoVP9(gof_);
1028 }
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001029 }
Alex Glaznevad948c42015-11-18 13:06:42 -08001030 picture_id_ = (picture_id_ + 1) & 0x7FFF;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001031
1032 // Generate a header describing a single fragment.
1033 webrtc::RTPFragmentationHeader header;
1034 memset(&header, 0, sizeof(header));
Alex Glaznevad948c42015-11-18 13:06:42 -08001035 if (codecType_ == kVideoCodecVP8 || codecType_ == kVideoCodecVP9) {
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001036 header.VerifyAndAllocateFragmentationHeader(1);
1037 header.fragmentationOffset[0] = 0;
1038 header.fragmentationLength[0] = image->_length;
1039 header.fragmentationPlType[0] = 0;
1040 header.fragmentationTimeDiff[0] = 0;
Alex Glaznevad948c42015-11-18 13:06:42 -08001041 if (codecType_ == kVideoCodecVP8 && scale_) {
asapersson86b01602015-10-20 23:55:26 -07001042 int qp;
glaznevf4decb52016-01-15 13:49:22 -08001043 if (webrtc::vp8::GetQp(payload, payload_size, &qp)) {
1044 current_acc_qp_ += qp;
asapersson86b01602015-10-20 23:55:26 -07001045 quality_scaler_.ReportQP(qp);
asapersson24ebc442016-04-19 23:48:21 -07001046 image->qp_ = qp;
glaznevf4decb52016-01-15 13:49:22 -08001047 }
asapersson86b01602015-10-20 23:55:26 -07001048 }
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001049 } else if (codecType_ == kVideoCodecH264) {
Peter Boström2bc68c72015-09-24 16:22:28 +02001050 if (scale_) {
1051 h264_bitstream_parser_.ParseBitstream(payload, payload_size);
1052 int qp;
glaznevf4decb52016-01-15 13:49:22 -08001053 if (h264_bitstream_parser_.GetLastSliceQp(&qp)) {
1054 current_acc_qp_ += qp;
Peter Boström2bc68c72015-09-24 16:22:28 +02001055 quality_scaler_.ReportQP(qp);
glaznevf4decb52016-01-15 13:49:22 -08001056 }
Peter Boström2bc68c72015-09-24 16:22:28 +02001057 }
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001058 // For H.264 search for start codes.
1059 int32_t scPositions[MAX_NALUS_PERFRAME + 1] = {};
1060 int32_t scPositionsLength = 0;
1061 int32_t scPosition = 0;
1062 while (scPositionsLength < MAX_NALUS_PERFRAME) {
1063 int32_t naluPosition = NextNaluPosition(
1064 payload + scPosition, payload_size - scPosition);
1065 if (naluPosition < 0) {
1066 break;
1067 }
1068 scPosition += naluPosition;
1069 scPositions[scPositionsLength++] = scPosition;
1070 scPosition += H264_SC_LENGTH;
1071 }
1072 if (scPositionsLength == 0) {
Alex Glaznevfddf6e52015-10-07 16:51:02 -07001073 ALOGE << "Start code is not found!";
1074 ALOGE << "Data:" << image->_buffer[0] << " " << image->_buffer[1]
1075 << " " << image->_buffer[2] << " " << image->_buffer[3]
1076 << " " << image->_buffer[4] << " " << image->_buffer[5];
perkj9576e542015-11-12 06:43:16 -08001077 ResetCodecOnCodecThread();
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001078 return false;
1079 }
1080 scPositions[scPositionsLength] = payload_size;
1081 header.VerifyAndAllocateFragmentationHeader(scPositionsLength);
1082 for (size_t i = 0; i < scPositionsLength; i++) {
1083 header.fragmentationOffset[i] = scPositions[i] + H264_SC_LENGTH;
1084 header.fragmentationLength[i] =
1085 scPositions[i + 1] - header.fragmentationOffset[i];
1086 header.fragmentationPlType[i] = 0;
1087 header.fragmentationTimeDiff[i] = 0;
1088 }
1089 }
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001090
1091 callback_status = callback_->Encoded(*image, &info, &header);
1092 }
1093
1094 // Return output buffer back to the encoder.
1095 bool success = jni->CallBooleanMethod(*j_media_codec_video_encoder_,
1096 j_release_output_buffer_method_,
1097 output_buffer_index);
1098 CHECK_EXCEPTION(jni);
1099 if (!success) {
perkj9576e542015-11-12 06:43:16 -08001100 ResetCodecOnCodecThread();
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001101 return false;
1102 }
1103
glaznevf4decb52016-01-15 13:49:22 -08001104 // Calculate and print encoding statistics - every 3 seconds.
1105 frames_encoded_++;
1106 current_frames_++;
1107 current_bytes_ += payload_size;
1108 current_encoding_time_ms_ += frame_encoding_time_ms;
glaznev94291482016-02-01 13:17:18 -08001109 LogStatistics(false);
glaznevf4decb52016-01-15 13:49:22 -08001110
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001111 if (callback_status > 0) {
1112 drop_next_input_frame_ = true;
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001113 // Theoretically could handle callback_status<0 here, but unclear what
1114 // that would mean for us.
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001115 }
1116 }
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001117 return true;
1118}
1119
glaznev94291482016-02-01 13:17:18 -08001120void MediaCodecVideoEncoder::LogStatistics(bool force_log) {
Niels Möllerd28db7f2016-05-10 16:31:47 +02001121 int statistic_time_ms = rtc::TimeMillis() - stat_start_time_ms_;
sakal1fc48102016-06-14 01:53:39 -07001122 if ((statistic_time_ms >= kMediaCodecStatisticsIntervalMs || force_log)
1123 && statistic_time_ms > 0) {
1124 // Prevent division by zero.
1125 int current_frames_divider = current_frames_ != 0 ? current_frames_ : 1;
1126
glaznev94291482016-02-01 13:17:18 -08001127 int current_bitrate = current_bytes_ * 8 / statistic_time_ms;
1128 int current_fps =
1129 (current_frames_ * 1000 + statistic_time_ms / 2) / statistic_time_ms;
1130 ALOGD << "Encoded frames: " << frames_encoded_ <<
1131 ". Bitrate: " << current_bitrate <<
1132 ", target: " << last_set_bitrate_kbps_ << " kbps" <<
1133 ", fps: " << current_fps <<
sakal1fc48102016-06-14 01:53:39 -07001134 ", encTime: " << (current_encoding_time_ms_ / current_frames_divider) <<
1135 ". QP: " << (current_acc_qp_ / current_frames_divider) <<
glaznev94291482016-02-01 13:17:18 -08001136 " for last " << statistic_time_ms << " ms.";
Niels Möllerd28db7f2016-05-10 16:31:47 +02001137 stat_start_time_ms_ = rtc::TimeMillis();
glaznev94291482016-02-01 13:17:18 -08001138 current_frames_ = 0;
1139 current_bytes_ = 0;
1140 current_acc_qp_ = 0;
1141 current_encoding_time_ms_ = 0;
1142 }
1143}
1144
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001145int32_t MediaCodecVideoEncoder::NextNaluPosition(
1146 uint8_t *buffer, size_t buffer_size) {
1147 if (buffer_size < H264_SC_LENGTH) {
1148 return -1;
1149 }
1150 uint8_t *head = buffer;
1151 // Set end buffer pointer to 4 bytes before actual buffer end so we can
1152 // access head[1], head[2] and head[3] in a loop without buffer overrun.
1153 uint8_t *end = buffer + buffer_size - H264_SC_LENGTH;
1154
1155 while (head < end) {
1156 if (head[0]) {
1157 head++;
1158 continue;
1159 }
1160 if (head[1]) { // got 00xx
1161 head += 2;
1162 continue;
1163 }
1164 if (head[2]) { // got 0000xx
1165 head += 3;
1166 continue;
1167 }
1168 if (head[3] != 0x01) { // got 000000xx
glaznev@webrtc.orgdc08a232015-03-06 23:32:20 +00001169 head++; // xx != 1, continue searching.
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001170 continue;
1171 }
1172 return (int32_t)(head - buffer);
1173 }
1174 return -1;
1175}
1176
jackychen61b4d512015-04-21 15:30:11 -07001177void MediaCodecVideoEncoder::OnDroppedFrame() {
Peter Boström53edac62016-04-27 00:08:34 +02001178 // Methods running on the codec thread should call OnDroppedFrameOnCodecThread
1179 // directly.
1180 RTC_DCHECK(!codec_thread_checker_.CalledOnValidThread());
1181 codec_thread_->Invoke<void>(
Taylor Brandstetter5d97a9a2016-06-10 14:17:27 -07001182 RTC_FROM_HERE,
Peter Boström53edac62016-04-27 00:08:34 +02001183 Bind(&MediaCodecVideoEncoder::OnDroppedFrameOnCodecThread, this));
1184}
1185
1186void MediaCodecVideoEncoder::OnDroppedFrameOnCodecThread() {
1187 RTC_DCHECK(codec_thread_checker_.CalledOnValidThread());
glaznevf4decb52016-01-15 13:49:22 -08001188 // Report dropped frame to quality_scaler_.
Peter Boström2bc68c72015-09-24 16:22:28 +02001189 if (scale_)
1190 quality_scaler_.ReportDroppedFrame();
jackychen61b4d512015-04-21 15:30:11 -07001191}
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001192
Peter Boströmb7d9a972015-12-18 16:01:11 +01001193const char* MediaCodecVideoEncoder::ImplementationName() const {
1194 return "MediaCodec";
1195}
1196
perkj461121c2016-02-15 06:28:36 -08001197MediaCodecVideoEncoderFactory::MediaCodecVideoEncoderFactory()
1198 : egl_context_(nullptr) {
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001199 JNIEnv* jni = AttachCurrentThreadIfNeeded();
1200 ScopedLocalRefFrame local_ref_frame(jni);
1201 jclass j_encoder_class = FindClass(jni, "org/webrtc/MediaCodecVideoEncoder");
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001202 supported_codecs_.clear();
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001203
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001204 bool is_vp8_hw_supported = jni->CallStaticBooleanMethod(
1205 j_encoder_class,
1206 GetStaticMethodID(jni, j_encoder_class, "isVp8HwSupported", "()Z"));
1207 CHECK_EXCEPTION(jni);
1208 if (is_vp8_hw_supported) {
Alex Glaznevfddf6e52015-10-07 16:51:02 -07001209 ALOGD << "VP8 HW Encoder supported.";
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001210 supported_codecs_.push_back(VideoCodec(kVideoCodecVP8, "VP8",
1211 MAX_VIDEO_WIDTH, MAX_VIDEO_HEIGHT, MAX_VIDEO_FPS));
1212 }
1213
Alex Glaznevad948c42015-11-18 13:06:42 -08001214 bool is_vp9_hw_supported = jni->CallStaticBooleanMethod(
1215 j_encoder_class,
1216 GetStaticMethodID(jni, j_encoder_class, "isVp9HwSupported", "()Z"));
1217 CHECK_EXCEPTION(jni);
1218 if (is_vp9_hw_supported) {
1219 ALOGD << "VP9 HW Encoder supported.";
1220 supported_codecs_.push_back(VideoCodec(kVideoCodecVP9, "VP9",
1221 MAX_VIDEO_WIDTH, MAX_VIDEO_HEIGHT, MAX_VIDEO_FPS));
1222 }
1223
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001224 bool is_h264_hw_supported = jni->CallStaticBooleanMethod(
1225 j_encoder_class,
1226 GetStaticMethodID(jni, j_encoder_class, "isH264HwSupported", "()Z"));
1227 CHECK_EXCEPTION(jni);
1228 if (is_h264_hw_supported) {
Alex Glaznevfddf6e52015-10-07 16:51:02 -07001229 ALOGD << "H.264 HW Encoder supported.";
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001230 supported_codecs_.push_back(VideoCodec(kVideoCodecH264, "H264",
1231 MAX_VIDEO_WIDTH, MAX_VIDEO_HEIGHT, MAX_VIDEO_FPS));
1232 }
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001233}
1234
Perec2922f2016-01-27 15:25:46 +01001235MediaCodecVideoEncoderFactory::~MediaCodecVideoEncoderFactory() {
1236 ALOGD << "MediaCodecVideoEncoderFactory dtor";
perkj461121c2016-02-15 06:28:36 -08001237 if (egl_context_) {
1238 JNIEnv* jni = AttachCurrentThreadIfNeeded();
1239 jni->DeleteGlobalRef(egl_context_);
1240 }
Perec2922f2016-01-27 15:25:46 +01001241}
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001242
perkj30e91822015-11-20 01:31:25 -08001243void MediaCodecVideoEncoderFactory::SetEGLContext(
perkj461121c2016-02-15 06:28:36 -08001244 JNIEnv* jni, jobject egl_context) {
perkj30e91822015-11-20 01:31:25 -08001245 ALOGD << "MediaCodecVideoEncoderFactory::SetEGLContext";
Perfd22e6c2016-02-18 11:35:48 +01001246 if (egl_context_) {
1247 jni->DeleteGlobalRef(egl_context_);
1248 egl_context_ = nullptr;
1249 }
perkj461121c2016-02-15 06:28:36 -08001250 egl_context_ = jni->NewGlobalRef(egl_context);
1251 if (CheckException(jni)) {
1252 ALOGE << "error calling NewGlobalRef for EGL Context.";
perkj30e91822015-11-20 01:31:25 -08001253 }
1254}
1255
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001256webrtc::VideoEncoder* MediaCodecVideoEncoderFactory::CreateVideoEncoder(
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001257 VideoCodecType type) {
1258 if (supported_codecs_.empty()) {
Alex Glaznevad948c42015-11-18 13:06:42 -08001259 ALOGW << "No HW video encoder for type " << (int)type;
Perec2922f2016-01-27 15:25:46 +01001260 return nullptr;
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001261 }
1262 for (std::vector<VideoCodec>::const_iterator it = supported_codecs_.begin();
1263 it != supported_codecs_.end(); ++it) {
1264 if (it->type == type) {
Alex Glaznevfddf6e52015-10-07 16:51:02 -07001265 ALOGD << "Create HW video encoder for type " << (int)type <<
1266 " (" << it->name << ").";
perkj30e91822015-11-20 01:31:25 -08001267 return new MediaCodecVideoEncoder(AttachCurrentThreadIfNeeded(), type,
perkj461121c2016-02-15 06:28:36 -08001268 egl_context_);
glaznev@webrtc.orgb28474c2015-02-23 17:44:27 +00001269 }
1270 }
Alex Glaznevad948c42015-11-18 13:06:42 -08001271 ALOGW << "Can not find HW video encoder for type " << (int)type;
Perec2922f2016-01-27 15:25:46 +01001272 return nullptr;
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001273}
1274
1275const std::vector<MediaCodecVideoEncoderFactory::VideoCodec>&
1276MediaCodecVideoEncoderFactory::codecs() const {
1277 return supported_codecs_;
1278}
1279
1280void MediaCodecVideoEncoderFactory::DestroyVideoEncoder(
1281 webrtc::VideoEncoder* encoder) {
Alex Glaznevfddf6e52015-10-07 16:51:02 -07001282 ALOGD << "Destroy video encoder.";
glaznev@webrtc.org18c92472015-02-18 18:42:55 +00001283 delete encoder;
1284}
1285
1286} // namespace webrtc_jni