blob: 0f92cfb5bdc81cf309b6239f2b06d6e932de0023 [file] [log] [blame]
Niels Möller530ead42018-10-04 14:28:39 +02001/*
2 * Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
11#include "audio/channel_receive.h"
12
13#include <algorithm>
14#include <map>
15#include <memory>
16#include <string>
17#include <utility>
18#include <vector>
19
20#include "absl/memory/memory.h"
Niels Möller349ade32018-11-16 09:50:42 +010021#include "audio/audio_level.h"
Niels Möller530ead42018-10-04 14:28:39 +020022#include "audio/channel_send.h"
23#include "audio/utility/audio_frame_operations.h"
24#include "logging/rtc_event_log/events/rtc_event_audio_playout.h"
25#include "logging/rtc_event_log/rtc_event_log.h"
26#include "modules/audio_coding/audio_network_adaptor/include/audio_network_adaptor_config.h"
Niels Möller349ade32018-11-16 09:50:42 +010027#include "modules/audio_coding/include/audio_coding_module.h"
Niels Möller530ead42018-10-04 14:28:39 +020028#include "modules/audio_device/include/audio_device.h"
29#include "modules/pacing/packet_router.h"
30#include "modules/rtp_rtcp/include/receive_statistics.h"
Niels Möller349ade32018-11-16 09:50:42 +010031#include "modules/rtp_rtcp/include/remote_ntp_time_estimator.h"
32#include "modules/rtp_rtcp/include/rtp_rtcp.h"
Artem Titov67008df2019-07-04 07:14:11 +000033#include "modules/rtp_rtcp/source/contributing_sources.h"
Yves Gerey988cc082018-10-23 12:03:01 +020034#include "modules/rtp_rtcp/source/rtp_header_extensions.h"
Niels Möller530ead42018-10-04 14:28:39 +020035#include "modules/rtp_rtcp/source/rtp_packet_received.h"
Danil Chapovalov2a977cf2018-12-04 18:03:52 +010036#include "modules/rtp_rtcp/source/rtp_rtcp_config.h"
Niels Möller530ead42018-10-04 14:28:39 +020037#include "modules/utility/include/process_thread.h"
38#include "rtc_base/checks.h"
Steve Anton10542f22019-01-11 09:11:00 -080039#include "rtc_base/critical_section.h"
Niels Möller530ead42018-10-04 14:28:39 +020040#include "rtc_base/format_macros.h"
41#include "rtc_base/location.h"
42#include "rtc_base/logging.h"
Niels Möller349ade32018-11-16 09:50:42 +010043#include "rtc_base/numerics/safe_minmax.h"
44#include "rtc_base/race_checker.h"
Niels Möller530ead42018-10-04 14:28:39 +020045#include "rtc_base/thread_checker.h"
Steve Anton10542f22019-01-11 09:11:00 -080046#include "rtc_base/time_utils.h"
Bjorn A Mellemda4f0932019-07-30 08:34:03 -070047#include "system_wrappers/include/field_trial.h"
Niels Möller530ead42018-10-04 14:28:39 +020048#include "system_wrappers/include/metrics.h"
49
50namespace webrtc {
51namespace voe {
52
53namespace {
54
55constexpr double kAudioSampleDurationSeconds = 0.01;
Niels Möller530ead42018-10-04 14:28:39 +020056
57// Video Sync.
58constexpr int kVoiceEngineMinMinPlayoutDelayMs = 0;
59constexpr int kVoiceEngineMaxMinPlayoutDelayMs = 10000;
60
Bjorn A Mellemda4f0932019-07-30 08:34:03 -070061// Field trial which controls whether to report standard-compliant bytes
62// sent/received per stream. If enabled, padding and headers are not included
63// in bytes sent or received.
64constexpr char kUseStandardBytesStats[] = "WebRTC-UseStandardBytesStats";
65
Niels Möllerafb5dbb2019-02-15 15:21:47 +010066RTPHeader CreateRTPHeaderForMediaTransportFrame(
Sergey Silkine049eba2019-02-18 09:52:26 +000067 const MediaTransportEncodedAudioFrame& frame,
68 uint64_t channel_id) {
Niels Möllerafb5dbb2019-02-15 15:21:47 +010069 webrtc::RTPHeader rtp_header;
70 rtp_header.payloadType = frame.payload_type();
71 rtp_header.payload_type_frequency = frame.sampling_rate_hz();
72 rtp_header.timestamp = frame.starting_sample_index();
73 rtp_header.sequenceNumber = frame.sequence_number();
Niels Möller7d76a312018-10-26 12:57:07 +020074
Sergey Silkine049eba2019-02-18 09:52:26 +000075 rtp_header.ssrc = static_cast<uint32_t>(channel_id);
Niels Möller7d76a312018-10-26 12:57:07 +020076
77 // The rest are initialized by the RTPHeader constructor.
Niels Möllerafb5dbb2019-02-15 15:21:47 +010078 return rtp_header;
Niels Möller7d76a312018-10-26 12:57:07 +020079}
80
Niels Möller349ade32018-11-16 09:50:42 +010081class ChannelReceive : public ChannelReceiveInterface,
82 public MediaTransportAudioSinkInterface {
83 public:
84 // Used for receive streams.
Sebastian Jansson977b3352019-03-04 17:43:34 +010085 ChannelReceive(Clock* clock,
86 ProcessThread* module_process_thread,
Niels Möller349ade32018-11-16 09:50:42 +010087 AudioDeviceModule* audio_device_module,
Anton Sukhanov4f08faa2019-05-21 11:12:57 -070088 const MediaTransportConfig& media_transport_config,
Niels Möller349ade32018-11-16 09:50:42 +010089 Transport* rtcp_send_transport,
90 RtcEventLog* rtc_event_log,
91 uint32_t remote_ssrc,
92 size_t jitter_buffer_max_packets,
93 bool jitter_buffer_fast_playout,
Jakob Ivarsson10403ae2018-11-27 15:45:20 +010094 int jitter_buffer_min_delay_ms,
Jakob Ivarsson53eae872019-01-10 15:58:36 +010095 bool jitter_buffer_enable_rtx_handling,
Niels Möller349ade32018-11-16 09:50:42 +010096 rtc::scoped_refptr<AudioDecoderFactory> decoder_factory,
97 absl::optional<AudioCodecPairId> codec_pair_id,
98 rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,
99 const webrtc::CryptoOptions& crypto_options);
100 ~ChannelReceive() override;
101
102 void SetSink(AudioSinkInterface* sink) override;
103
104 void SetReceiveCodecs(const std::map<int, SdpAudioFormat>& codecs) override;
105
106 // API methods
107
108 void StartPlayout() override;
109 void StopPlayout() override;
110
111 // Codecs
Benjamin Wright2af5dcb2019-04-09 20:08:41 +0000112 absl::optional<std::pair<int, SdpAudioFormat>> GetReceiveCodec()
113 const override;
Niels Möller349ade32018-11-16 09:50:42 +0100114
Niels Möller8fb1a6a2019-03-05 14:29:42 +0100115 void ReceivedRTCPPacket(const uint8_t* data, size_t length) override;
Niels Möller349ade32018-11-16 09:50:42 +0100116
117 // RtpPacketSinkInterface.
118 void OnRtpPacket(const RtpPacketReceived& packet) override;
119
120 // Muting, Volume and Level.
121 void SetChannelOutputVolumeScaling(float scaling) override;
122 int GetSpeechOutputLevelFullRange() const override;
123 // See description of "totalAudioEnergy" in the WebRTC stats spec:
124 // https://w3c.github.io/webrtc-stats/#dom-rtcmediastreamtrackstats-totalaudioenergy
125 double GetTotalOutputEnergy() const override;
126 double GetTotalOutputDuration() const override;
127
128 // Stats.
129 NetworkStatistics GetNetworkStatistics() const override;
130 AudioDecodingCallStats GetDecodingCallStatistics() const override;
131
132 // Audio+Video Sync.
133 uint32_t GetDelayEstimate() const override;
134 void SetMinimumPlayoutDelay(int delayMs) override;
135 uint32_t GetPlayoutTimestamp() const override;
136
Ruslan Burakov3b50f9f2019-02-06 09:45:56 +0100137 // Audio quality.
138 bool SetBaseMinimumPlayoutDelayMs(int delay_ms) override;
139 int GetBaseMinimumPlayoutDelayMs() const override;
140
Niels Möller349ade32018-11-16 09:50:42 +0100141 // Produces the transport-related timestamps; current_delay_ms is left unset.
142 absl::optional<Syncable::Info> GetSyncInfo() const override;
143
144 // RTP+RTCP
145 void SetLocalSSRC(unsigned int ssrc) override;
146
147 void RegisterReceiverCongestionControlObjects(
148 PacketRouter* packet_router) override;
149 void ResetReceiverCongestionControlObjects() override;
150
151 CallReceiveStatistics GetRTCPStatistics() const override;
152 void SetNACKStatus(bool enable, int maxNumberOfPackets) override;
153
154 AudioMixer::Source::AudioFrameInfo GetAudioFrameWithInfo(
155 int sample_rate_hz,
156 AudioFrame* audio_frame) override;
157
158 int PreferredSampleRate() const override;
159
160 // Associate to a send channel.
161 // Used for obtaining RTT for a receive-only channel.
Niels Möllerdced9f62018-11-19 10:27:07 +0100162 void SetAssociatedSendChannel(const ChannelSendInterface* channel) override;
Niels Möller349ade32018-11-16 09:50:42 +0100163
Artem Titov67008df2019-07-04 07:14:11 +0000164 std::vector<RtpSource> GetSources() const override;
165
Anton Sukhanov4f08faa2019-05-21 11:12:57 -0700166 // TODO(sukhanov): Return const pointer. It requires making media transport
167 // getters like GetLatestTargetTransferRate to be also const.
168 MediaTransportInterface* media_transport() const {
169 return media_transport_config_.media_transport;
170 }
171
Niels Möller349ade32018-11-16 09:50:42 +0100172 private:
Niels Möller349ade32018-11-16 09:50:42 +0100173 bool ReceivePacket(const uint8_t* packet,
174 size_t packet_length,
175 const RTPHeader& header);
176 int ResendPackets(const uint16_t* sequence_numbers, int length);
177 void UpdatePlayoutTimestamp(bool rtcp);
178
179 int GetRtpTimestampRateHz() const;
180 int64_t GetRTT() const;
181
182 // MediaTransportAudioSinkInterface override;
Sergey Silkine049eba2019-02-18 09:52:26 +0000183 void OnData(uint64_t channel_id,
184 MediaTransportEncodedAudioFrame frame) override;
Niels Möller349ade32018-11-16 09:50:42 +0100185
186 int32_t OnReceivedPayloadData(const uint8_t* payloadData,
187 size_t payloadSize,
Niels Möllerafb5dbb2019-02-15 15:21:47 +0100188 const RTPHeader& rtpHeader);
Niels Möller349ade32018-11-16 09:50:42 +0100189
Fredrik Solenbergc5e8be32018-11-19 11:56:13 +0100190 bool Playing() const {
191 rtc::CritScope lock(&playing_lock_);
192 return playing_;
193 }
194
Niels Möller349ade32018-11-16 09:50:42 +0100195 // Thread checkers document and lock usage of some methods to specific threads
196 // we know about. The goal is to eventually split up voe::ChannelReceive into
197 // parts with single-threaded semantics, and thereby reduce the need for
198 // locks.
199 rtc::ThreadChecker worker_thread_checker_;
200 rtc::ThreadChecker module_process_thread_checker_;
201 // Methods accessed from audio and video threads are checked for sequential-
202 // only access. We don't necessarily own and control these threads, so thread
203 // checkers cannot be used. E.g. Chromium may transfer "ownership" from one
204 // audio thread to another, but access is still sequential.
205 rtc::RaceChecker audio_thread_race_checker_;
206 rtc::RaceChecker video_capture_thread_race_checker_;
207 rtc::CriticalSection _callbackCritSect;
208 rtc::CriticalSection volume_settings_critsect_;
209
Fredrik Solenbergc5e8be32018-11-19 11:56:13 +0100210 rtc::CriticalSection playing_lock_;
211 bool playing_ RTC_GUARDED_BY(&playing_lock_) = false;
Niels Möller349ade32018-11-16 09:50:42 +0100212
213 RtcEventLog* const event_log_;
214
215 // Indexed by payload type.
216 std::map<uint8_t, int> payload_type_frequencies_;
217
218 std::unique_ptr<ReceiveStatistics> rtp_receive_statistics_;
219 std::unique_ptr<RtpRtcp> _rtpRtcpModule;
220 const uint32_t remote_ssrc_;
221
Artem Titov67008df2019-07-04 07:14:11 +0000222 // Info for GetSources and GetSyncInfo is updated on network or worker thread,
223 // queried on the worker thread.
224 rtc::CriticalSection rtp_sources_lock_;
225 ContributingSources contributing_sources_ RTC_GUARDED_BY(&rtp_sources_lock_);
Niels Möller349ade32018-11-16 09:50:42 +0100226 absl::optional<uint32_t> last_received_rtp_timestamp_
Artem Titov67008df2019-07-04 07:14:11 +0000227 RTC_GUARDED_BY(&rtp_sources_lock_);
Niels Möller349ade32018-11-16 09:50:42 +0100228 absl::optional<int64_t> last_received_rtp_system_time_ms_
Artem Titov67008df2019-07-04 07:14:11 +0000229 RTC_GUARDED_BY(&rtp_sources_lock_);
230 absl::optional<uint8_t> last_received_rtp_audio_level_
231 RTC_GUARDED_BY(&rtp_sources_lock_);
Niels Möller349ade32018-11-16 09:50:42 +0100232
233 std::unique_ptr<AudioCodingModule> audio_coding_;
234 AudioSinkInterface* audio_sink_ = nullptr;
235 AudioLevel _outputAudioLevel;
236
237 RemoteNtpTimeEstimator ntp_estimator_ RTC_GUARDED_BY(ts_stats_lock_);
238
239 // Timestamp of the audio pulled from NetEq.
240 absl::optional<uint32_t> jitter_buffer_playout_timestamp_;
241
242 rtc::CriticalSection video_sync_lock_;
243 uint32_t playout_timestamp_rtp_ RTC_GUARDED_BY(video_sync_lock_);
244 uint32_t playout_delay_ms_ RTC_GUARDED_BY(video_sync_lock_);
245
246 rtc::CriticalSection ts_stats_lock_;
247
248 std::unique_ptr<rtc::TimestampWrapAroundHandler> rtp_ts_wraparound_handler_;
249 // The rtp timestamp of the first played out audio frame.
250 int64_t capture_start_rtp_time_stamp_;
251 // The capture ntp time (in local timebase) of the first played out audio
252 // frame.
253 int64_t capture_start_ntp_time_ms_ RTC_GUARDED_BY(ts_stats_lock_);
254
255 // uses
256 ProcessThread* _moduleProcessThreadPtr;
257 AudioDeviceModule* _audioDeviceModulePtr;
258 float _outputGain RTC_GUARDED_BY(volume_settings_critsect_);
259
260 // An associated send channel.
261 rtc::CriticalSection assoc_send_channel_lock_;
Niels Möllerdced9f62018-11-19 10:27:07 +0100262 const ChannelSendInterface* associated_send_channel_
Niels Möller349ade32018-11-16 09:50:42 +0100263 RTC_GUARDED_BY(assoc_send_channel_lock_);
264
265 PacketRouter* packet_router_ = nullptr;
266
267 rtc::ThreadChecker construction_thread_;
268
Anton Sukhanov4f08faa2019-05-21 11:12:57 -0700269 MediaTransportConfig media_transport_config_;
Niels Möller349ade32018-11-16 09:50:42 +0100270
271 // E2EE Audio Frame Decryption
272 rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor_;
273 webrtc::CryptoOptions crypto_options_;
Bjorn A Mellemda4f0932019-07-30 08:34:03 -0700274
275 const bool use_standard_bytes_stats_;
Niels Möller349ade32018-11-16 09:50:42 +0100276};
Niels Möller530ead42018-10-04 14:28:39 +0200277
Niels Möllerafb5dbb2019-02-15 15:21:47 +0100278int32_t ChannelReceive::OnReceivedPayloadData(const uint8_t* payloadData,
279 size_t payloadSize,
280 const RTPHeader& rtp_header) {
Niels Möller7d76a312018-10-26 12:57:07 +0200281 // We should not be receiving any RTP packets if media_transport is set.
Anton Sukhanov4f08faa2019-05-21 11:12:57 -0700282 RTC_CHECK(!media_transport());
Niels Möller7d76a312018-10-26 12:57:07 +0200283
Fredrik Solenbergc5e8be32018-11-19 11:56:13 +0100284 if (!Playing()) {
Niels Möller530ead42018-10-04 14:28:39 +0200285 // Avoid inserting into NetEQ when we are not playing. Count the
286 // packet as discarded.
287 return 0;
288 }
289
290 // Push the incoming payload (parsed and ready for decoding) into the ACM
Niels Möllerafb5dbb2019-02-15 15:21:47 +0100291 if (audio_coding_->IncomingPacket(payloadData, payloadSize, rtp_header) !=
Niels Möller530ead42018-10-04 14:28:39 +0200292 0) {
293 RTC_DLOG(LS_ERROR) << "ChannelReceive::OnReceivedPayloadData() unable to "
294 "push data to the ACM";
295 return -1;
296 }
297
298 int64_t round_trip_time = 0;
299 _rtpRtcpModule->RTT(remote_ssrc_, &round_trip_time, NULL, NULL, NULL);
300
301 std::vector<uint16_t> nack_list = audio_coding_->GetNackList(round_trip_time);
302 if (!nack_list.empty()) {
303 // Can't use nack_list.data() since it's not supported by all
304 // compilers.
305 ResendPackets(&(nack_list[0]), static_cast<int>(nack_list.size()));
306 }
307 return 0;
308}
309
Niels Möller7d76a312018-10-26 12:57:07 +0200310// MediaTransportAudioSinkInterface override.
Sergey Silkine049eba2019-02-18 09:52:26 +0000311void ChannelReceive::OnData(uint64_t channel_id,
312 MediaTransportEncodedAudioFrame frame) {
Anton Sukhanov4f08faa2019-05-21 11:12:57 -0700313 RTC_CHECK(media_transport());
Niels Möller7d76a312018-10-26 12:57:07 +0200314
Fredrik Solenbergc5e8be32018-11-19 11:56:13 +0100315 if (!Playing()) {
Niels Möller7d76a312018-10-26 12:57:07 +0200316 // Avoid inserting into NetEQ when we are not playing. Count the
317 // packet as discarded.
318 return;
319 }
320
321 // Send encoded audio frame to Decoder / NetEq.
322 if (audio_coding_->IncomingPacket(
323 frame.encoded_data().data(), frame.encoded_data().size(),
Sergey Silkine049eba2019-02-18 09:52:26 +0000324 CreateRTPHeaderForMediaTransportFrame(frame, channel_id)) != 0) {
Niels Möller7d76a312018-10-26 12:57:07 +0200325 RTC_DLOG(LS_ERROR) << "ChannelReceive::OnData: unable to "
326 "push data to the ACM";
327 }
328}
329
Niels Möller530ead42018-10-04 14:28:39 +0200330AudioMixer::Source::AudioFrameInfo ChannelReceive::GetAudioFrameWithInfo(
331 int sample_rate_hz,
332 AudioFrame* audio_frame) {
Niels Möller349ade32018-11-16 09:50:42 +0100333 RTC_DCHECK_RUNS_SERIALIZED(&audio_thread_race_checker_);
Niels Möller530ead42018-10-04 14:28:39 +0200334 audio_frame->sample_rate_hz_ = sample_rate_hz;
335
Fredrik Solenbergc5e8be32018-11-19 11:56:13 +0100336 event_log_->Log(absl::make_unique<RtcEventAudioPlayout>(remote_ssrc_));
337
Niels Möller530ead42018-10-04 14:28:39 +0200338 // Get 10ms raw PCM data from the ACM (mixer limits output frequency)
339 bool muted;
340 if (audio_coding_->PlayoutData10Ms(audio_frame->sample_rate_hz_, audio_frame,
341 &muted) == -1) {
342 RTC_DLOG(LS_ERROR)
343 << "ChannelReceive::GetAudioFrame() PlayoutData10Ms() failed!";
344 // In all likelihood, the audio in this frame is garbage. We return an
345 // error so that the audio mixer module doesn't add it to the mix. As
346 // a result, it won't be played out and the actions skipped here are
347 // irrelevant.
348 return AudioMixer::Source::AudioFrameInfo::kError;
349 }
350
351 if (muted) {
352 // TODO(henrik.lundin): We should be able to do better than this. But we
353 // will have to go through all the cases below where the audio samples may
354 // be used, and handle the muted case in some way.
355 AudioFrameOperations::Mute(audio_frame);
356 }
357
358 {
359 // Pass the audio buffers to an optional sink callback, before applying
360 // scaling/panning, as that applies to the mix operation.
361 // External recipients of the audio (e.g. via AudioTrack), will do their
362 // own mixing/dynamic processing.
363 rtc::CritScope cs(&_callbackCritSect);
364 if (audio_sink_) {
365 AudioSinkInterface::Data data(
366 audio_frame->data(), audio_frame->samples_per_channel_,
367 audio_frame->sample_rate_hz_, audio_frame->num_channels_,
368 audio_frame->timestamp_);
369 audio_sink_->OnData(data);
370 }
371 }
372
373 float output_gain = 1.0f;
374 {
375 rtc::CritScope cs(&volume_settings_critsect_);
376 output_gain = _outputGain;
377 }
378
379 // Output volume scaling
380 if (output_gain < 0.99f || output_gain > 1.01f) {
381 // TODO(solenberg): Combine with mute state - this can cause clicks!
382 AudioFrameOperations::ScaleWithSat(output_gain, audio_frame);
383 }
384
385 // Measure audio level (0-9)
386 // TODO(henrik.lundin) Use the |muted| information here too.
387 // TODO(deadbeef): Use RmsLevel for |_outputAudioLevel| (see
388 // https://crbug.com/webrtc/7517).
389 _outputAudioLevel.ComputeLevel(*audio_frame, kAudioSampleDurationSeconds);
390
391 if (capture_start_rtp_time_stamp_ < 0 && audio_frame->timestamp_ != 0) {
392 // The first frame with a valid rtp timestamp.
393 capture_start_rtp_time_stamp_ = audio_frame->timestamp_;
394 }
395
396 if (capture_start_rtp_time_stamp_ >= 0) {
397 // audio_frame.timestamp_ should be valid from now on.
398
399 // Compute elapsed time.
400 int64_t unwrap_timestamp =
401 rtp_ts_wraparound_handler_->Unwrap(audio_frame->timestamp_);
402 audio_frame->elapsed_time_ms_ =
403 (unwrap_timestamp - capture_start_rtp_time_stamp_) /
404 (GetRtpTimestampRateHz() / 1000);
405
406 {
407 rtc::CritScope lock(&ts_stats_lock_);
408 // Compute ntp time.
409 audio_frame->ntp_time_ms_ =
410 ntp_estimator_.Estimate(audio_frame->timestamp_);
411 // |ntp_time_ms_| won't be valid until at least 2 RTCP SRs are received.
412 if (audio_frame->ntp_time_ms_ > 0) {
413 // Compute |capture_start_ntp_time_ms_| so that
414 // |capture_start_ntp_time_ms_| + |elapsed_time_ms_| == |ntp_time_ms_|
415 capture_start_ntp_time_ms_ =
416 audio_frame->ntp_time_ms_ - audio_frame->elapsed_time_ms_;
417 }
418 }
419 }
420
421 {
422 RTC_HISTOGRAM_COUNTS_1000("WebRTC.Audio.TargetJitterBufferDelayMs",
423 audio_coding_->TargetDelayMs());
424 const int jitter_buffer_delay = audio_coding_->FilteredCurrentDelayMs();
425 rtc::CritScope lock(&video_sync_lock_);
426 RTC_HISTOGRAM_COUNTS_1000("WebRTC.Audio.ReceiverDelayEstimateMs",
427 jitter_buffer_delay + playout_delay_ms_);
428 RTC_HISTOGRAM_COUNTS_1000("WebRTC.Audio.ReceiverJitterBufferDelayMs",
429 jitter_buffer_delay);
430 RTC_HISTOGRAM_COUNTS_1000("WebRTC.Audio.ReceiverDeviceDelayMs",
431 playout_delay_ms_);
432 }
433
434 return muted ? AudioMixer::Source::AudioFrameInfo::kMuted
435 : AudioMixer::Source::AudioFrameInfo::kNormal;
436}
437
438int ChannelReceive::PreferredSampleRate() const {
Niels Möller349ade32018-11-16 09:50:42 +0100439 RTC_DCHECK_RUNS_SERIALIZED(&audio_thread_race_checker_);
Niels Möller530ead42018-10-04 14:28:39 +0200440 // Return the bigger of playout and receive frequency in the ACM.
441 return std::max(audio_coding_->ReceiveFrequency(),
442 audio_coding_->PlayoutFrequency());
443}
444
445ChannelReceive::ChannelReceive(
Sebastian Jansson977b3352019-03-04 17:43:34 +0100446 Clock* clock,
Niels Möller530ead42018-10-04 14:28:39 +0200447 ProcessThread* module_process_thread,
448 AudioDeviceModule* audio_device_module,
Anton Sukhanov4f08faa2019-05-21 11:12:57 -0700449 const MediaTransportConfig& media_transport_config,
Niels Möllerae4237e2018-10-05 11:28:38 +0200450 Transport* rtcp_send_transport,
Niels Möller530ead42018-10-04 14:28:39 +0200451 RtcEventLog* rtc_event_log,
452 uint32_t remote_ssrc,
453 size_t jitter_buffer_max_packets,
454 bool jitter_buffer_fast_playout,
Jakob Ivarsson10403ae2018-11-27 15:45:20 +0100455 int jitter_buffer_min_delay_ms,
Jakob Ivarsson53eae872019-01-10 15:58:36 +0100456 bool jitter_buffer_enable_rtx_handling,
Niels Möller530ead42018-10-04 14:28:39 +0200457 rtc::scoped_refptr<AudioDecoderFactory> decoder_factory,
Benjamin Wright84583f62018-10-04 14:22:34 -0700458 absl::optional<AudioCodecPairId> codec_pair_id,
Benjamin Wright78410ad2018-10-25 09:52:57 -0700459 rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,
Benjamin Wrightbfb444c2018-10-15 10:20:24 -0700460 const webrtc::CryptoOptions& crypto_options)
Niels Möller530ead42018-10-04 14:28:39 +0200461 : event_log_(rtc_event_log),
Sebastian Jansson977b3352019-03-04 17:43:34 +0100462 rtp_receive_statistics_(ReceiveStatistics::Create(clock)),
Niels Möller530ead42018-10-04 14:28:39 +0200463 remote_ssrc_(remote_ssrc),
464 _outputAudioLevel(),
Sebastian Jansson977b3352019-03-04 17:43:34 +0100465 ntp_estimator_(clock),
Niels Möller530ead42018-10-04 14:28:39 +0200466 playout_timestamp_rtp_(0),
467 playout_delay_ms_(0),
468 rtp_ts_wraparound_handler_(new rtc::TimestampWrapAroundHandler()),
469 capture_start_rtp_time_stamp_(-1),
470 capture_start_ntp_time_ms_(-1),
471 _moduleProcessThreadPtr(module_process_thread),
472 _audioDeviceModulePtr(audio_device_module),
Niels Möller530ead42018-10-04 14:28:39 +0200473 _outputGain(1.0f),
Benjamin Wright84583f62018-10-04 14:22:34 -0700474 associated_send_channel_(nullptr),
Anton Sukhanov4f08faa2019-05-21 11:12:57 -0700475 media_transport_config_(media_transport_config),
Benjamin Wrightbfb444c2018-10-15 10:20:24 -0700476 frame_decryptor_(frame_decryptor),
Bjorn A Mellemda4f0932019-07-30 08:34:03 -0700477 crypto_options_(crypto_options),
478 use_standard_bytes_stats_(
479 webrtc::field_trial::IsEnabled(kUseStandardBytesStats)) {
Niels Möller349ade32018-11-16 09:50:42 +0100480 // TODO(nisse): Use _moduleProcessThreadPtr instead?
Sebastian Janssonc01367d2019-04-08 15:20:44 +0200481 module_process_thread_checker_.Detach();
Niels Möller349ade32018-11-16 09:50:42 +0100482
Niels Möller530ead42018-10-04 14:28:39 +0200483 RTC_DCHECK(module_process_thread);
484 RTC_DCHECK(audio_device_module);
485 AudioCodingModule::Config acm_config;
486 acm_config.decoder_factory = decoder_factory;
487 acm_config.neteq_config.codec_pair_id = codec_pair_id;
488 acm_config.neteq_config.max_packets_in_buffer = jitter_buffer_max_packets;
489 acm_config.neteq_config.enable_fast_accelerate = jitter_buffer_fast_playout;
Jakob Ivarsson10403ae2018-11-27 15:45:20 +0100490 acm_config.neteq_config.min_delay_ms = jitter_buffer_min_delay_ms;
Niels Möller530ead42018-10-04 14:28:39 +0200491 acm_config.neteq_config.enable_muted_state = true;
Jakob Ivarsson53eae872019-01-10 15:58:36 +0100492 acm_config.neteq_config.enable_rtx_handling =
493 jitter_buffer_enable_rtx_handling;
Niels Möller530ead42018-10-04 14:28:39 +0200494 audio_coding_.reset(AudioCodingModule::Create(acm_config));
495
Henrik Boströmd2c336f2019-07-03 17:11:10 +0200496 _outputAudioLevel.ResetLevelFullRange();
Niels Möller530ead42018-10-04 14:28:39 +0200497
498 rtp_receive_statistics_->EnableRetransmitDetection(remote_ssrc_, true);
499 RtpRtcp::Configuration configuration;
Sebastian Jansson977b3352019-03-04 17:43:34 +0100500 configuration.clock = clock;
Niels Möller530ead42018-10-04 14:28:39 +0200501 configuration.audio = true;
Niels Möllerfd1a2fb2018-10-31 15:25:26 +0100502 configuration.receiver_only = true;
Niels Möllerae4237e2018-10-05 11:28:38 +0200503 configuration.outgoing_transport = rtcp_send_transport;
Niels Möller530ead42018-10-04 14:28:39 +0200504 configuration.receive_statistics = rtp_receive_statistics_.get();
505
506 configuration.event_log = event_log_;
Niels Möller530ead42018-10-04 14:28:39 +0200507
Danil Chapovalovc44f6cc2019-03-06 11:31:09 +0100508 _rtpRtcpModule = RtpRtcp::Create(configuration);
Niels Möller530ead42018-10-04 14:28:39 +0200509 _rtpRtcpModule->SetSendingMediaStatus(false);
510 _rtpRtcpModule->SetRemoteSSRC(remote_ssrc_);
Niels Möller530ead42018-10-04 14:28:39 +0200511
Niels Möller530ead42018-10-04 14:28:39 +0200512 _moduleProcessThreadPtr->RegisterModule(_rtpRtcpModule.get(), RTC_FROM_HERE);
513
Niels Möller530ead42018-10-04 14:28:39 +0200514 // Ensure that RTCP is enabled by default for the created channel.
515 // Note that, the module will keep generating RTCP until it is explicitly
516 // disabled by the user.
517 // After StopListen (when no sockets exists), RTCP packets will no longer
518 // be transmitted since the Transport object will then be invalid.
519 // RTCP is enabled by default.
520 _rtpRtcpModule->SetRTCPStatus(RtcpMode::kCompound);
Niels Möller7d76a312018-10-26 12:57:07 +0200521
Anton Sukhanov4f08faa2019-05-21 11:12:57 -0700522 if (media_transport()) {
523 media_transport()->SetReceiveAudioSink(this);
Niels Möller7d76a312018-10-26 12:57:07 +0200524 }
Niels Möller530ead42018-10-04 14:28:39 +0200525}
526
Fredrik Solenbergc5e8be32018-11-19 11:56:13 +0100527ChannelReceive::~ChannelReceive() {
Sebastian Janssonc01367d2019-04-08 15:20:44 +0200528 RTC_DCHECK(construction_thread_.IsCurrent());
Niels Möller7d76a312018-10-26 12:57:07 +0200529
Anton Sukhanov4f08faa2019-05-21 11:12:57 -0700530 if (media_transport()) {
531 media_transport()->SetReceiveAudioSink(nullptr);
Niels Möller7d76a312018-10-26 12:57:07 +0200532 }
533
Niels Möller530ead42018-10-04 14:28:39 +0200534 StopPlayout();
535
Niels Möller530ead42018-10-04 14:28:39 +0200536 int error = audio_coding_->RegisterTransportCallback(NULL);
537 RTC_DCHECK_EQ(0, error);
538
Niels Möller530ead42018-10-04 14:28:39 +0200539 if (_moduleProcessThreadPtr)
540 _moduleProcessThreadPtr->DeRegisterModule(_rtpRtcpModule.get());
Niels Möller530ead42018-10-04 14:28:39 +0200541}
542
543void ChannelReceive::SetSink(AudioSinkInterface* sink) {
Sebastian Janssonc01367d2019-04-08 15:20:44 +0200544 RTC_DCHECK(worker_thread_checker_.IsCurrent());
Niels Möller530ead42018-10-04 14:28:39 +0200545 rtc::CritScope cs(&_callbackCritSect);
546 audio_sink_ = sink;
547}
548
Niels Möller80c67622018-11-12 13:22:47 +0100549void ChannelReceive::StartPlayout() {
Sebastian Janssonc01367d2019-04-08 15:20:44 +0200550 RTC_DCHECK(worker_thread_checker_.IsCurrent());
Fredrik Solenbergc5e8be32018-11-19 11:56:13 +0100551 rtc::CritScope lock(&playing_lock_);
552 playing_ = true;
Niels Möller530ead42018-10-04 14:28:39 +0200553}
554
Niels Möller80c67622018-11-12 13:22:47 +0100555void ChannelReceive::StopPlayout() {
Sebastian Janssonc01367d2019-04-08 15:20:44 +0200556 RTC_DCHECK(worker_thread_checker_.IsCurrent());
Fredrik Solenbergc5e8be32018-11-19 11:56:13 +0100557 rtc::CritScope lock(&playing_lock_);
558 playing_ = false;
Henrik Boströmd2c336f2019-07-03 17:11:10 +0200559 _outputAudioLevel.ResetLevelFullRange();
Niels Möller530ead42018-10-04 14:28:39 +0200560}
561
Jonas Olssona4d87372019-07-05 19:08:33 +0200562absl::optional<std::pair<int, SdpAudioFormat>> ChannelReceive::GetReceiveCodec()
563 const {
Sebastian Janssonc01367d2019-04-08 15:20:44 +0200564 RTC_DCHECK(worker_thread_checker_.IsCurrent());
Fredrik Solenbergf693bfa2018-12-11 12:22:10 +0100565 return audio_coding_->ReceiveCodec();
Niels Möller530ead42018-10-04 14:28:39 +0200566}
567
Artem Titov67008df2019-07-04 07:14:11 +0000568std::vector<webrtc::RtpSource> ChannelReceive::GetSources() const {
569 RTC_DCHECK(worker_thread_checker_.IsCurrent());
570 int64_t now_ms = rtc::TimeMillis();
571 std::vector<RtpSource> sources;
572 {
573 rtc::CritScope cs(&rtp_sources_lock_);
574 sources = contributing_sources_.GetSources(now_ms);
575 if (last_received_rtp_system_time_ms_ >=
576 now_ms - ContributingSources::kHistoryMs) {
577 RTC_DCHECK(last_received_rtp_timestamp_.has_value());
578 sources.emplace_back(*last_received_rtp_system_time_ms_, remote_ssrc_,
579 RtpSourceType::SSRC, last_received_rtp_audio_level_,
580 *last_received_rtp_timestamp_);
581 }
582 }
583 return sources;
584}
585
Niels Möller530ead42018-10-04 14:28:39 +0200586void ChannelReceive::SetReceiveCodecs(
587 const std::map<int, SdpAudioFormat>& codecs) {
Sebastian Janssonc01367d2019-04-08 15:20:44 +0200588 RTC_DCHECK(worker_thread_checker_.IsCurrent());
Niels Möller530ead42018-10-04 14:28:39 +0200589 for (const auto& kv : codecs) {
590 RTC_DCHECK_GE(kv.second.clockrate_hz, 1000);
591 payload_type_frequencies_[kv.first] = kv.second.clockrate_hz;
592 }
593 audio_coding_->SetReceiveCodecs(codecs);
594}
595
Niels Möller349ade32018-11-16 09:50:42 +0100596// May be called on either worker thread or network thread.
Niels Möller530ead42018-10-04 14:28:39 +0200597void ChannelReceive::OnRtpPacket(const RtpPacketReceived& packet) {
598 int64_t now_ms = rtc::TimeMillis();
Artem Titov67008df2019-07-04 07:14:11 +0000599 uint8_t audio_level;
600 bool voice_activity;
601 bool has_audio_level =
602 packet.GetExtension<::webrtc::AudioLevel>(&voice_activity, &audio_level);
Niels Möller530ead42018-10-04 14:28:39 +0200603
604 {
Artem Titov67008df2019-07-04 07:14:11 +0000605 rtc::CritScope cs(&rtp_sources_lock_);
Niels Möller530ead42018-10-04 14:28:39 +0200606 last_received_rtp_timestamp_ = packet.Timestamp();
607 last_received_rtp_system_time_ms_ = now_ms;
Artem Titov67008df2019-07-04 07:14:11 +0000608 if (has_audio_level)
609 last_received_rtp_audio_level_ = audio_level;
610 std::vector<uint32_t> csrcs = packet.Csrcs();
611 contributing_sources_.Update(
612 now_ms, csrcs,
613 has_audio_level ? absl::optional<uint8_t>(audio_level) : absl::nullopt,
614 packet.Timestamp());
Niels Möller530ead42018-10-04 14:28:39 +0200615 }
616
617 // Store playout timestamp for the received RTP packet
618 UpdatePlayoutTimestamp(false);
619
620 const auto& it = payload_type_frequencies_.find(packet.PayloadType());
621 if (it == payload_type_frequencies_.end())
622 return;
623 // TODO(nisse): Set payload_type_frequency earlier, when packet is parsed.
624 RtpPacketReceived packet_copy(packet);
625 packet_copy.set_payload_type_frequency(it->second);
626
627 rtp_receive_statistics_->OnRtpPacket(packet_copy);
628
629 RTPHeader header;
630 packet_copy.GetHeader(&header);
631
632 ReceivePacket(packet_copy.data(), packet_copy.size(), header);
633}
634
635bool ChannelReceive::ReceivePacket(const uint8_t* packet,
636 size_t packet_length,
637 const RTPHeader& header) {
638 const uint8_t* payload = packet + header.headerLength;
639 assert(packet_length >= header.headerLength);
640 size_t payload_length = packet_length - header.headerLength;
Niels Möller530ead42018-10-04 14:28:39 +0200641
Benjamin Wright84583f62018-10-04 14:22:34 -0700642 size_t payload_data_length = payload_length - header.paddingLength;
643
644 // E2EE Custom Audio Frame Decryption (This is optional).
645 // Keep this buffer around for the lifetime of the OnReceivedPayloadData call.
646 rtc::Buffer decrypted_audio_payload;
647 if (frame_decryptor_ != nullptr) {
Benjamin Wright2af5dcb2019-04-09 20:08:41 +0000648 const size_t max_plaintext_size = frame_decryptor_->GetMaxPlaintextByteSize(
Benjamin Wright84583f62018-10-04 14:22:34 -0700649 cricket::MEDIA_TYPE_AUDIO, payload_length);
650 decrypted_audio_payload.SetSize(max_plaintext_size);
651
Benjamin Wright2af5dcb2019-04-09 20:08:41 +0000652 const std::vector<uint32_t> csrcs(header.arrOfCSRCs,
653 header.arrOfCSRCs + header.numCSRCs);
654 const FrameDecryptorInterface::Result decrypt_result =
655 frame_decryptor_->Decrypt(
656 cricket::MEDIA_TYPE_AUDIO, csrcs,
657 /*additional_data=*/nullptr,
658 rtc::ArrayView<const uint8_t>(payload, payload_data_length),
659 decrypted_audio_payload);
Benjamin Wright84583f62018-10-04 14:22:34 -0700660
Benjamin Wright2af5dcb2019-04-09 20:08:41 +0000661 if (decrypt_result.IsOk()) {
662 decrypted_audio_payload.SetSize(decrypt_result.bytes_written);
663 } else {
664 // Interpret failures as a silent frame.
665 decrypted_audio_payload.SetSize(0);
Benjamin Wright84583f62018-10-04 14:22:34 -0700666 }
667
Benjamin Wright84583f62018-10-04 14:22:34 -0700668 payload = decrypted_audio_payload.data();
669 payload_data_length = decrypted_audio_payload.size();
Benjamin Wrightbfb444c2018-10-15 10:20:24 -0700670 } else if (crypto_options_.sframe.require_frame_encryption) {
671 RTC_DLOG(LS_ERROR)
672 << "FrameDecryptor required but not set, dropping packet";
673 payload_data_length = 0;
Benjamin Wright84583f62018-10-04 14:22:34 -0700674 }
675
Niels Möller530ead42018-10-04 14:28:39 +0200676 if (payload_data_length == 0) {
Niels Möllerafb5dbb2019-02-15 15:21:47 +0100677 return OnReceivedPayloadData(nullptr, 0, header);
Niels Möller530ead42018-10-04 14:28:39 +0200678 }
Niels Möllerafb5dbb2019-02-15 15:21:47 +0100679 return OnReceivedPayloadData(payload, payload_data_length, header);
Niels Möller530ead42018-10-04 14:28:39 +0200680}
681
Niels Möller349ade32018-11-16 09:50:42 +0100682// May be called on either worker thread or network thread.
Niels Möller8fb1a6a2019-03-05 14:29:42 +0100683void ChannelReceive::ReceivedRTCPPacket(const uint8_t* data, size_t length) {
Niels Möller530ead42018-10-04 14:28:39 +0200684 // Store playout timestamp for the received RTCP packet
685 UpdatePlayoutTimestamp(true);
686
687 // Deliver RTCP packet to RTP/RTCP module for parsing
688 _rtpRtcpModule->IncomingRtcpPacket(data, length);
689
690 int64_t rtt = GetRTT();
691 if (rtt == 0) {
692 // Waiting for valid RTT.
Niels Möller8fb1a6a2019-03-05 14:29:42 +0100693 return;
Niels Möller530ead42018-10-04 14:28:39 +0200694 }
695
Niels Möller530ead42018-10-04 14:28:39 +0200696 uint32_t ntp_secs = 0;
697 uint32_t ntp_frac = 0;
698 uint32_t rtp_timestamp = 0;
699 if (0 != _rtpRtcpModule->RemoteNTP(&ntp_secs, &ntp_frac, NULL, NULL,
700 &rtp_timestamp)) {
701 // Waiting for RTCP.
Niels Möller8fb1a6a2019-03-05 14:29:42 +0100702 return;
Niels Möller530ead42018-10-04 14:28:39 +0200703 }
704
705 {
706 rtc::CritScope lock(&ts_stats_lock_);
707 ntp_estimator_.UpdateRtcpTimestamp(rtt, ntp_secs, ntp_frac, rtp_timestamp);
708 }
Niels Möller530ead42018-10-04 14:28:39 +0200709}
710
711int ChannelReceive::GetSpeechOutputLevelFullRange() const {
Sebastian Janssonc01367d2019-04-08 15:20:44 +0200712 RTC_DCHECK(worker_thread_checker_.IsCurrent());
Niels Möller530ead42018-10-04 14:28:39 +0200713 return _outputAudioLevel.LevelFullRange();
714}
715
716double ChannelReceive::GetTotalOutputEnergy() const {
Sebastian Janssonc01367d2019-04-08 15:20:44 +0200717 RTC_DCHECK(worker_thread_checker_.IsCurrent());
Niels Möller530ead42018-10-04 14:28:39 +0200718 return _outputAudioLevel.TotalEnergy();
719}
720
721double ChannelReceive::GetTotalOutputDuration() const {
Sebastian Janssonc01367d2019-04-08 15:20:44 +0200722 RTC_DCHECK(worker_thread_checker_.IsCurrent());
Niels Möller530ead42018-10-04 14:28:39 +0200723 return _outputAudioLevel.TotalDuration();
724}
725
726void ChannelReceive::SetChannelOutputVolumeScaling(float scaling) {
Sebastian Janssonc01367d2019-04-08 15:20:44 +0200727 RTC_DCHECK(worker_thread_checker_.IsCurrent());
Niels Möller530ead42018-10-04 14:28:39 +0200728 rtc::CritScope cs(&volume_settings_critsect_);
729 _outputGain = scaling;
730}
731
Niels Möller349ade32018-11-16 09:50:42 +0100732void ChannelReceive::SetLocalSSRC(uint32_t ssrc) {
Sebastian Janssonc01367d2019-04-08 15:20:44 +0200733 RTC_DCHECK(worker_thread_checker_.IsCurrent());
Niels Möller530ead42018-10-04 14:28:39 +0200734 _rtpRtcpModule->SetSSRC(ssrc);
Niels Möller530ead42018-10-04 14:28:39 +0200735}
736
Niels Möller530ead42018-10-04 14:28:39 +0200737void ChannelReceive::RegisterReceiverCongestionControlObjects(
738 PacketRouter* packet_router) {
Sebastian Janssonc01367d2019-04-08 15:20:44 +0200739 RTC_DCHECK(worker_thread_checker_.IsCurrent());
Niels Möller530ead42018-10-04 14:28:39 +0200740 RTC_DCHECK(packet_router);
741 RTC_DCHECK(!packet_router_);
742 constexpr bool remb_candidate = false;
743 packet_router->AddReceiveRtpModule(_rtpRtcpModule.get(), remb_candidate);
744 packet_router_ = packet_router;
745}
746
747void ChannelReceive::ResetReceiverCongestionControlObjects() {
Sebastian Janssonc01367d2019-04-08 15:20:44 +0200748 RTC_DCHECK(worker_thread_checker_.IsCurrent());
Niels Möller530ead42018-10-04 14:28:39 +0200749 RTC_DCHECK(packet_router_);
750 packet_router_->RemoveReceiveRtpModule(_rtpRtcpModule.get());
751 packet_router_ = nullptr;
752}
753
Niels Möller349ade32018-11-16 09:50:42 +0100754CallReceiveStatistics ChannelReceive::GetRTCPStatistics() const {
Sebastian Janssonc01367d2019-04-08 15:20:44 +0200755 RTC_DCHECK(worker_thread_checker_.IsCurrent());
Niels Möller530ead42018-10-04 14:28:39 +0200756 // --- RtcpStatistics
Niels Möller80c67622018-11-12 13:22:47 +0100757 CallReceiveStatistics stats;
Niels Möller530ead42018-10-04 14:28:39 +0200758
759 // The jitter statistics is updated for each received RTP packet and is
760 // based on received packets.
761 RtcpStatistics statistics;
762 StreamStatistician* statistician =
763 rtp_receive_statistics_->GetStatistician(remote_ssrc_);
764 if (statistician) {
765 statistician->GetStatistics(&statistics,
766 _rtpRtcpModule->RTCP() == RtcpMode::kOff);
767 }
768
Niels Möller530ead42018-10-04 14:28:39 +0200769 stats.cumulativeLost = statistics.packets_lost;
770 stats.extendedMax = statistics.extended_highest_sequence_number;
771 stats.jitterSamples = statistics.jitter;
772
773 // --- RTT
774 stats.rttMs = GetRTT();
775
776 // --- Data counters
Niels Möller530ead42018-10-04 14:28:39 +0200777 if (statistician) {
Henrik Boström01738c62019-04-15 17:32:00 +0200778 StreamDataCounters data_counters;
779 statistician->GetReceiveStreamDataCounters(&data_counters);
Bjorn A Mellemda4f0932019-07-30 08:34:03 -0700780 if (use_standard_bytes_stats_) {
781 stats.bytesReceived = data_counters.transmitted.payload_bytes;
782 } else {
783 stats.bytesReceived = data_counters.transmitted.payload_bytes +
784 data_counters.transmitted.header_bytes +
785 data_counters.transmitted.padding_bytes;
786 }
Henrik Boström01738c62019-04-15 17:32:00 +0200787 stats.packetsReceived = data_counters.transmitted.packets;
788 stats.last_packet_received_timestamp_ms =
789 data_counters.last_packet_received_timestamp_ms;
790 } else {
791 stats.bytesReceived = 0;
792 stats.packetsReceived = 0;
793 stats.last_packet_received_timestamp_ms = absl::nullopt;
Niels Möller530ead42018-10-04 14:28:39 +0200794 }
795
Niels Möller530ead42018-10-04 14:28:39 +0200796 // --- Timestamps
797 {
798 rtc::CritScope lock(&ts_stats_lock_);
799 stats.capture_start_ntp_time_ms_ = capture_start_ntp_time_ms_;
800 }
Niels Möller80c67622018-11-12 13:22:47 +0100801 return stats;
Niels Möller530ead42018-10-04 14:28:39 +0200802}
803
Niels Möller349ade32018-11-16 09:50:42 +0100804void ChannelReceive::SetNACKStatus(bool enable, int max_packets) {
Sebastian Janssonc01367d2019-04-08 15:20:44 +0200805 RTC_DCHECK(worker_thread_checker_.IsCurrent());
Niels Möller530ead42018-10-04 14:28:39 +0200806 // None of these functions can fail.
Danil Chapovalov2a977cf2018-12-04 18:03:52 +0100807 if (enable) {
Niels Möller87da1092019-05-24 14:04:28 +0200808 rtp_receive_statistics_->SetMaxReorderingThreshold(remote_ssrc_,
809 max_packets);
Niels Möller349ade32018-11-16 09:50:42 +0100810 audio_coding_->EnableNack(max_packets);
Danil Chapovalov2a977cf2018-12-04 18:03:52 +0100811 } else {
812 rtp_receive_statistics_->SetMaxReorderingThreshold(
Niels Möller87da1092019-05-24 14:04:28 +0200813 remote_ssrc_, kDefaultMaxReorderingThreshold);
Niels Möller530ead42018-10-04 14:28:39 +0200814 audio_coding_->DisableNack();
Danil Chapovalov2a977cf2018-12-04 18:03:52 +0100815 }
Niels Möller530ead42018-10-04 14:28:39 +0200816}
817
818// Called when we are missing one or more packets.
819int ChannelReceive::ResendPackets(const uint16_t* sequence_numbers,
820 int length) {
821 return _rtpRtcpModule->SendNACK(sequence_numbers, length);
822}
823
Niels Möllerdced9f62018-11-19 10:27:07 +0100824void ChannelReceive::SetAssociatedSendChannel(
825 const ChannelSendInterface* channel) {
Sebastian Janssonc01367d2019-04-08 15:20:44 +0200826 RTC_DCHECK(worker_thread_checker_.IsCurrent());
Niels Möller530ead42018-10-04 14:28:39 +0200827 rtc::CritScope lock(&assoc_send_channel_lock_);
828 associated_send_channel_ = channel;
829}
830
Niels Möller80c67622018-11-12 13:22:47 +0100831NetworkStatistics ChannelReceive::GetNetworkStatistics() const {
Sebastian Janssonc01367d2019-04-08 15:20:44 +0200832 RTC_DCHECK(worker_thread_checker_.IsCurrent());
Niels Möller80c67622018-11-12 13:22:47 +0100833 NetworkStatistics stats;
834 int error = audio_coding_->GetNetworkStatistics(&stats);
835 RTC_DCHECK_EQ(0, error);
836 return stats;
Niels Möller530ead42018-10-04 14:28:39 +0200837}
838
Niels Möller80c67622018-11-12 13:22:47 +0100839AudioDecodingCallStats ChannelReceive::GetDecodingCallStatistics() const {
Sebastian Janssonc01367d2019-04-08 15:20:44 +0200840 RTC_DCHECK(worker_thread_checker_.IsCurrent());
Niels Möller80c67622018-11-12 13:22:47 +0100841 AudioDecodingCallStats stats;
842 audio_coding_->GetDecodingCallStatistics(&stats);
843 return stats;
Niels Möller530ead42018-10-04 14:28:39 +0200844}
845
846uint32_t ChannelReceive::GetDelayEstimate() const {
Sebastian Janssonc01367d2019-04-08 15:20:44 +0200847 RTC_DCHECK(worker_thread_checker_.IsCurrent() ||
848 module_process_thread_checker_.IsCurrent());
Niels Möller530ead42018-10-04 14:28:39 +0200849 rtc::CritScope lock(&video_sync_lock_);
850 return audio_coding_->FilteredCurrentDelayMs() + playout_delay_ms_;
851}
852
Niels Möller349ade32018-11-16 09:50:42 +0100853void ChannelReceive::SetMinimumPlayoutDelay(int delay_ms) {
Sebastian Janssonc01367d2019-04-08 15:20:44 +0200854 RTC_DCHECK(module_process_thread_checker_.IsCurrent());
Niels Möller349ade32018-11-16 09:50:42 +0100855 // Limit to range accepted by both VoE and ACM, so we're at least getting as
856 // close as possible, instead of failing.
Ruslan Burakov432c8332019-02-03 22:21:02 +0100857 delay_ms = rtc::SafeClamp(delay_ms, kVoiceEngineMinMinPlayoutDelayMs,
858 kVoiceEngineMaxMinPlayoutDelayMs);
Niels Möller349ade32018-11-16 09:50:42 +0100859 if (audio_coding_->SetMinimumPlayoutDelay(delay_ms) != 0) {
Niels Möller530ead42018-10-04 14:28:39 +0200860 RTC_DLOG(LS_ERROR)
861 << "SetMinimumPlayoutDelay() failed to set min playout delay";
Niels Möller530ead42018-10-04 14:28:39 +0200862 }
Niels Möller530ead42018-10-04 14:28:39 +0200863}
864
Niels Möller349ade32018-11-16 09:50:42 +0100865uint32_t ChannelReceive::GetPlayoutTimestamp() const {
866 RTC_DCHECK_RUNS_SERIALIZED(&video_capture_thread_race_checker_);
Niels Möller530ead42018-10-04 14:28:39 +0200867 {
868 rtc::CritScope lock(&video_sync_lock_);
Niels Möller80c67622018-11-12 13:22:47 +0100869 return playout_timestamp_rtp_;
Niels Möller530ead42018-10-04 14:28:39 +0200870 }
Niels Möller530ead42018-10-04 14:28:39 +0200871}
872
Ruslan Burakov3b50f9f2019-02-06 09:45:56 +0100873bool ChannelReceive::SetBaseMinimumPlayoutDelayMs(int delay_ms) {
874 return audio_coding_->SetBaseMinimumPlayoutDelayMs(delay_ms);
875}
876
877int ChannelReceive::GetBaseMinimumPlayoutDelayMs() const {
878 return audio_coding_->GetBaseMinimumPlayoutDelayMs();
879}
880
Niels Möller530ead42018-10-04 14:28:39 +0200881absl::optional<Syncable::Info> ChannelReceive::GetSyncInfo() const {
Sebastian Janssonc01367d2019-04-08 15:20:44 +0200882 RTC_DCHECK(module_process_thread_checker_.IsCurrent());
Niels Möller530ead42018-10-04 14:28:39 +0200883 Syncable::Info info;
884 if (_rtpRtcpModule->RemoteNTP(&info.capture_time_ntp_secs,
885 &info.capture_time_ntp_frac, nullptr, nullptr,
886 &info.capture_time_source_clock) != 0) {
887 return absl::nullopt;
888 }
889 {
Artem Titov67008df2019-07-04 07:14:11 +0000890 rtc::CritScope cs(&rtp_sources_lock_);
Niels Möller530ead42018-10-04 14:28:39 +0200891 if (!last_received_rtp_timestamp_ || !last_received_rtp_system_time_ms_) {
892 return absl::nullopt;
893 }
894 info.latest_received_capture_timestamp = *last_received_rtp_timestamp_;
895 info.latest_receive_time_ms = *last_received_rtp_system_time_ms_;
896 }
897 return info;
898}
899
900void ChannelReceive::UpdatePlayoutTimestamp(bool rtcp) {
901 jitter_buffer_playout_timestamp_ = audio_coding_->PlayoutTimestamp();
902
903 if (!jitter_buffer_playout_timestamp_) {
904 // This can happen if this channel has not received any RTP packets. In
905 // this case, NetEq is not capable of computing a playout timestamp.
906 return;
907 }
908
909 uint16_t delay_ms = 0;
910 if (_audioDeviceModulePtr->PlayoutDelay(&delay_ms) == -1) {
911 RTC_DLOG(LS_WARNING)
912 << "ChannelReceive::UpdatePlayoutTimestamp() failed to read"
913 << " playout delay from the ADM";
914 return;
915 }
916
917 RTC_DCHECK(jitter_buffer_playout_timestamp_);
918 uint32_t playout_timestamp = *jitter_buffer_playout_timestamp_;
919
920 // Remove the playout delay.
921 playout_timestamp -= (delay_ms * (GetRtpTimestampRateHz() / 1000));
922
923 {
924 rtc::CritScope lock(&video_sync_lock_);
925 if (!rtcp) {
926 playout_timestamp_rtp_ = playout_timestamp;
927 }
928 playout_delay_ms_ = delay_ms;
929 }
930}
931
932int ChannelReceive::GetRtpTimestampRateHz() const {
Fredrik Solenbergf693bfa2018-12-11 12:22:10 +0100933 const auto decoder = audio_coding_->ReceiveCodec();
Niels Möller530ead42018-10-04 14:28:39 +0200934 // Default to the playout frequency if we've not gotten any packets yet.
935 // TODO(ossu): Zero clockrate can only happen if we've added an external
936 // decoder for a format we don't support internally. Remove once that way of
937 // adding decoders is gone!
Fredrik Solenbergf693bfa2018-12-11 12:22:10 +0100938 return (decoder && decoder->second.clockrate_hz != 0)
939 ? decoder->second.clockrate_hz
Niels Möller530ead42018-10-04 14:28:39 +0200940 : audio_coding_->PlayoutFrequency();
941}
942
943int64_t ChannelReceive::GetRTT() const {
Anton Sukhanov4f08faa2019-05-21 11:12:57 -0700944 if (media_transport()) {
945 auto target_rate = media_transport()->GetLatestTargetTransferRate();
Piotr (Peter) Slatala179a3922018-11-16 09:57:58 -0800946 if (target_rate.has_value()) {
947 return target_rate->network_estimate.round_trip_time.ms();
948 }
949
950 return 0;
951 }
Niels Möller530ead42018-10-04 14:28:39 +0200952 RtcpMode method = _rtpRtcpModule->RTCP();
953 if (method == RtcpMode::kOff) {
954 return 0;
955 }
956 std::vector<RTCPReportBlock> report_blocks;
957 _rtpRtcpModule->RemoteRTCPStat(&report_blocks);
958
959 // TODO(nisse): Could we check the return value from the ->RTT() call below,
960 // instead of checking if we have any report blocks?
961 if (report_blocks.empty()) {
962 rtc::CritScope lock(&assoc_send_channel_lock_);
963 // Tries to get RTT from an associated channel.
964 if (!associated_send_channel_) {
965 return 0;
966 }
967 return associated_send_channel_->GetRTT();
968 }
969
970 int64_t rtt = 0;
971 int64_t avg_rtt = 0;
972 int64_t max_rtt = 0;
973 int64_t min_rtt = 0;
Niels Möllerfd1a2fb2018-10-31 15:25:26 +0100974 // TODO(nisse): This method computes RTT based on sender reports, even though
975 // a receive stream is not supposed to do that.
Niels Möller530ead42018-10-04 14:28:39 +0200976 if (_rtpRtcpModule->RTT(remote_ssrc_, &rtt, &avg_rtt, &min_rtt, &max_rtt) !=
977 0) {
978 return 0;
979 }
980 return rtt;
981}
982
Niels Möller349ade32018-11-16 09:50:42 +0100983} // namespace
984
985std::unique_ptr<ChannelReceiveInterface> CreateChannelReceive(
Sebastian Jansson977b3352019-03-04 17:43:34 +0100986 Clock* clock,
Niels Möller349ade32018-11-16 09:50:42 +0100987 ProcessThread* module_process_thread,
988 AudioDeviceModule* audio_device_module,
Anton Sukhanov4f08faa2019-05-21 11:12:57 -0700989 const MediaTransportConfig& media_transport_config,
Niels Möller349ade32018-11-16 09:50:42 +0100990 Transport* rtcp_send_transport,
991 RtcEventLog* rtc_event_log,
992 uint32_t remote_ssrc,
993 size_t jitter_buffer_max_packets,
994 bool jitter_buffer_fast_playout,
Jakob Ivarsson10403ae2018-11-27 15:45:20 +0100995 int jitter_buffer_min_delay_ms,
Jakob Ivarsson53eae872019-01-10 15:58:36 +0100996 bool jitter_buffer_enable_rtx_handling,
Niels Möller349ade32018-11-16 09:50:42 +0100997 rtc::scoped_refptr<AudioDecoderFactory> decoder_factory,
998 absl::optional<AudioCodecPairId> codec_pair_id,
999 rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,
1000 const webrtc::CryptoOptions& crypto_options) {
1001 return absl::make_unique<ChannelReceive>(
Anton Sukhanov4f08faa2019-05-21 11:12:57 -07001002 clock, module_process_thread, audio_device_module, media_transport_config,
Niels Möller349ade32018-11-16 09:50:42 +01001003 rtcp_send_transport, rtc_event_log, remote_ssrc,
Jakob Ivarsson10403ae2018-11-27 15:45:20 +01001004 jitter_buffer_max_packets, jitter_buffer_fast_playout,
Jakob Ivarsson53eae872019-01-10 15:58:36 +01001005 jitter_buffer_min_delay_ms, jitter_buffer_enable_rtx_handling,
1006 decoder_factory, codec_pair_id, frame_decryptor, crypto_options);
Niels Möller349ade32018-11-16 09:50:42 +01001007}
1008
Niels Möller530ead42018-10-04 14:28:39 +02001009} // namespace voe
1010} // namespace webrtc