blob: 7895af3bb7677e9293c5f0bf497f6069c4cba04f [file] [log] [blame]
solenbergc7a8b082015-10-16 14:35:07 -07001/*
2 * Copyright (c) 2015 The WebRTC project authors. All Rights Reserved.
3 *
4 * Use of this source code is governed by a BSD-style license
5 * that can be found in the LICENSE file in the root of the source
6 * tree. An additional intellectual property rights grant can be found
7 * in the file PATENTS. All contributing project authors may
8 * be found in the AUTHORS file in the root of the source tree.
9 */
10
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020011#include "audio/audio_send_stream.h"
solenbergc7a8b082015-10-16 14:35:07 -070012
13#include <string>
ossu20a4b3f2017-04-27 02:08:52 -070014#include <utility>
15#include <vector>
solenbergc7a8b082015-10-16 14:35:07 -070016
Mirko Bonadei92ea95e2017-09-15 06:47:31 +020017#include "audio/audio_state.h"
18#include "audio/conversion.h"
19#include "audio/scoped_voe_interface.h"
20#include "call/rtp_transport_controller_send_interface.h"
21#include "modules/audio_coding/codecs/cng/audio_encoder_cng.h"
22#include "modules/bitrate_controller/include/bitrate_controller.h"
23#include "modules/congestion_controller/include/send_side_congestion_controller.h"
24#include "modules/pacing/paced_sender.h"
25#include "rtc_base/checks.h"
26#include "rtc_base/event.h"
27#include "rtc_base/function_view.h"
28#include "rtc_base/logging.h"
29#include "rtc_base/task_queue.h"
30#include "rtc_base/timeutils.h"
31#include "voice_engine/channel_proxy.h"
32#include "voice_engine/include/voe_base.h"
33#include "voice_engine/transmit_mixer.h"
34#include "voice_engine/voice_engine_impl.h"
solenbergc7a8b082015-10-16 14:35:07 -070035
36namespace webrtc {
minyue7a973442016-10-20 03:27:12 -070037
solenbergc7a8b082015-10-16 14:35:07 -070038namespace internal {
eladalonedd6eea2017-05-25 00:15:35 -070039// TODO(eladalon): Subsequent CL will make these values experiment-dependent.
elad.alond12a8e12017-03-23 11:04:48 -070040constexpr size_t kPacketLossTrackerMaxWindowSizeMs = 15000;
41constexpr size_t kPacketLossRateMinNumAckedPackets = 50;
42constexpr size_t kRecoverablePacketLossRateMinNumAckedPairs = 40;
43
ossu20a4b3f2017-04-27 02:08:52 -070044namespace {
45void CallEncoder(const std::unique_ptr<voe::ChannelProxy>& channel_proxy,
46 rtc::FunctionView<void(AudioEncoder*)> lambda) {
47 channel_proxy->ModifyEncoder([&](std::unique_ptr<AudioEncoder>* encoder_ptr) {
48 RTC_DCHECK(encoder_ptr);
49 lambda(encoder_ptr->get());
50 });
51}
52} // namespace
53
sazac58f8c02017-07-19 00:39:19 -070054// TODO(saza): Move this declaration further down when we can use
55// std::make_unique.
56class AudioSendStream::TimedTransport : public Transport {
57 public:
58 TimedTransport(Transport* transport, TimeInterval* time_interval)
59 : transport_(transport), lifetime_(time_interval) {}
60 bool SendRtp(const uint8_t* packet,
61 size_t length,
62 const PacketOptions& options) {
63 if (lifetime_) {
64 lifetime_->Extend();
65 }
66 return transport_->SendRtp(packet, length, options);
67 }
68 bool SendRtcp(const uint8_t* packet, size_t length) {
69 return transport_->SendRtcp(packet, length);
70 }
71 ~TimedTransport() {}
72
73 private:
74 Transport* transport_;
75 TimeInterval* lifetime_;
76};
77
solenberg566ef242015-11-06 15:34:49 -080078AudioSendStream::AudioSendStream(
79 const webrtc::AudioSendStream::Config& config,
Stefan Holmerb86d4e42015-12-07 10:26:18 +010080 const rtc::scoped_refptr<webrtc::AudioState>& audio_state,
perkj26091b12016-09-01 01:17:40 -070081 rtc::TaskQueue* worker_queue,
nisseb8f9a322017-03-27 05:36:15 -070082 RtpTransportControllerSendInterface* transport,
tereliuse035e2d2016-09-21 06:51:47 -070083 BitrateAllocator* bitrate_allocator,
michaelt9332b7d2016-11-30 07:51:13 -080084 RtcEventLog* event_log,
ossuc3d4b482017-05-23 06:07:11 -070085 RtcpRttStats* rtcp_rtt_stats,
86 const rtc::Optional<RtpState>& suspended_rtp_state)
perkj26091b12016-09-01 01:17:40 -070087 : worker_queue_(worker_queue),
ossu20a4b3f2017-04-27 02:08:52 -070088 config_(Config(nullptr)),
mflodman86cc6ff2016-07-26 04:44:06 -070089 audio_state_(audio_state),
ossu20a4b3f2017-04-27 02:08:52 -070090 event_log_(event_log),
michaeltf4caaab2017-01-16 23:55:07 -080091 bitrate_allocator_(bitrate_allocator),
nisseb8f9a322017-03-27 05:36:15 -070092 transport_(transport),
elad.alond12a8e12017-03-23 11:04:48 -070093 packet_loss_tracker_(kPacketLossTrackerMaxWindowSizeMs,
94 kPacketLossRateMinNumAckedPackets,
ossuc3d4b482017-05-23 06:07:11 -070095 kRecoverablePacketLossRateMinNumAckedPairs),
96 rtp_rtcp_module_(nullptr),
97 suspended_rtp_state_(suspended_rtp_state) {
ossu20a4b3f2017-04-27 02:08:52 -070098 LOG(LS_INFO) << "AudioSendStream: " << config.ToString();
99 RTC_DCHECK_NE(config.voe_channel_id, -1);
solenberg566ef242015-11-06 15:34:49 -0800100 RTC_DCHECK(audio_state_.get());
nisseb8f9a322017-03-27 05:36:15 -0700101 RTC_DCHECK(transport);
102 RTC_DCHECK(transport->send_side_cc());
solenberg3a941542015-11-16 07:34:50 -0800103
solenberg13725082015-11-25 08:16:52 -0800104 VoiceEngineImpl* voe_impl = static_cast<VoiceEngineImpl*>(voice_engine());
ossu20a4b3f2017-04-27 02:08:52 -0700105 channel_proxy_ = voe_impl->GetChannelProxy(config.voe_channel_id);
106 channel_proxy_->SetRtcEventLog(event_log_);
michaelt9332b7d2016-11-30 07:51:13 -0800107 channel_proxy_->SetRtcpRttStats(rtcp_rtt_stats);
solenberg13725082015-11-25 08:16:52 -0800108 channel_proxy_->SetRTCPStatus(true);
ossuc3d4b482017-05-23 06:07:11 -0700109 RtpReceiver* rtpReceiver = nullptr; // Unused, but required for call.
110 channel_proxy_->GetRtpRtcp(&rtp_rtcp_module_, &rtpReceiver);
111 RTC_DCHECK(rtp_rtcp_module_);
mflodman3d7db262016-04-29 00:57:13 -0700112
ossu20a4b3f2017-04-27 02:08:52 -0700113 ConfigureStream(this, config, true);
elad.alond12a8e12017-03-23 11:04:48 -0700114
115 pacer_thread_checker_.DetachFromThread();
Danil Chapovalov90e1f532017-10-03 14:59:27 +0200116 // Signal congestion controller this object is ready for OnPacket* callbacks.
117 transport_->send_side_cc()->RegisterPacketFeedbackObserver(this);
solenbergc7a8b082015-10-16 14:35:07 -0700118}
119
120AudioSendStream::~AudioSendStream() {
elad.alond12a8e12017-03-23 11:04:48 -0700121 RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
solenbergc7a8b082015-10-16 14:35:07 -0700122 LOG(LS_INFO) << "~AudioSendStream: " << config_.ToString();
nisseb8f9a322017-03-27 05:36:15 -0700123 transport_->send_side_cc()->DeRegisterPacketFeedbackObserver(this);
solenberg1c239d42017-09-29 06:00:28 -0700124 channel_proxy_->RegisterTransport(nullptr);
nissefdbfdc92017-03-31 05:44:52 -0700125 channel_proxy_->ResetSenderCongestionControlObjects();
tereliuse035e2d2016-09-21 06:51:47 -0700126 channel_proxy_->SetRtcEventLog(nullptr);
michaelt9332b7d2016-11-30 07:51:13 -0800127 channel_proxy_->SetRtcpRttStats(nullptr);
solenbergc7a8b082015-10-16 14:35:07 -0700128}
129
eladalonabbc4302017-07-26 02:09:44 -0700130const webrtc::AudioSendStream::Config& AudioSendStream::GetConfig() const {
131 RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
132 return config_;
133}
134
ossu20a4b3f2017-04-27 02:08:52 -0700135void AudioSendStream::Reconfigure(
136 const webrtc::AudioSendStream::Config& new_config) {
137 ConfigureStream(this, new_config, false);
138}
139
140void AudioSendStream::ConfigureStream(
141 webrtc::internal::AudioSendStream* stream,
142 const webrtc::AudioSendStream::Config& new_config,
143 bool first_time) {
144 LOG(LS_INFO) << "AudioSendStream::Configuring: " << new_config.ToString();
145 const auto& channel_proxy = stream->channel_proxy_;
146 const auto& old_config = stream->config_;
147
148 if (first_time || old_config.rtp.ssrc != new_config.rtp.ssrc) {
149 channel_proxy->SetLocalSSRC(new_config.rtp.ssrc);
ossuc3d4b482017-05-23 06:07:11 -0700150 if (stream->suspended_rtp_state_) {
151 stream->rtp_rtcp_module_->SetRtpState(*stream->suspended_rtp_state_);
152 }
ossu20a4b3f2017-04-27 02:08:52 -0700153 }
154 if (first_time || old_config.rtp.c_name != new_config.rtp.c_name) {
155 channel_proxy->SetRTCP_CNAME(new_config.rtp.c_name);
156 }
157 // TODO(solenberg): Config NACK history window (which is a packet count),
158 // using the actual packet size for the configured codec.
159 if (first_time || old_config.rtp.nack.rtp_history_ms !=
160 new_config.rtp.nack.rtp_history_ms) {
161 channel_proxy->SetNACKStatus(new_config.rtp.nack.rtp_history_ms != 0,
162 new_config.rtp.nack.rtp_history_ms / 20);
163 }
164
165 if (first_time ||
166 new_config.send_transport != old_config.send_transport) {
167 if (old_config.send_transport) {
solenberg1c239d42017-09-29 06:00:28 -0700168 channel_proxy->RegisterTransport(nullptr);
ossu20a4b3f2017-04-27 02:08:52 -0700169 }
sazac58f8c02017-07-19 00:39:19 -0700170 if (new_config.send_transport) {
171 stream->timed_send_transport_adapter_.reset(new TimedTransport(
172 new_config.send_transport, &stream->active_lifetime_));
173 } else {
174 stream->timed_send_transport_adapter_.reset(nullptr);
175 }
solenberg1c239d42017-09-29 06:00:28 -0700176 channel_proxy->RegisterTransport(
sazac58f8c02017-07-19 00:39:19 -0700177 stream->timed_send_transport_adapter_.get());
ossu20a4b3f2017-04-27 02:08:52 -0700178 }
179
180 // RFC 5285: Each distinct extension MUST have a unique ID. The value 0 is
181 // reserved for padding and MUST NOT be used as a local identifier.
182 // So it should be safe to use 0 here to indicate "not configured".
183 struct ExtensionIds {
184 int audio_level = 0;
185 int transport_sequence_number = 0;
186 };
187
188 auto find_extension_ids = [](const std::vector<RtpExtension>& extensions) {
189 ExtensionIds ids;
190 for (const auto& extension : extensions) {
191 if (extension.uri == RtpExtension::kAudioLevelUri) {
192 ids.audio_level = extension.id;
193 } else if (extension.uri == RtpExtension::kTransportSequenceNumberUri) {
194 ids.transport_sequence_number = extension.id;
195 }
196 }
197 return ids;
198 };
199
200 const ExtensionIds old_ids = find_extension_ids(old_config.rtp.extensions);
201 const ExtensionIds new_ids = find_extension_ids(new_config.rtp.extensions);
202 // Audio level indication
203 if (first_time || new_ids.audio_level != old_ids.audio_level) {
204 channel_proxy->SetSendAudioLevelIndicationStatus(new_ids.audio_level != 0,
205 new_ids.audio_level);
206 }
207 // Transport sequence number
208 if (first_time ||
209 new_ids.transport_sequence_number != old_ids.transport_sequence_number) {
ossu1129df22017-06-30 01:38:56 -0700210 if (!first_time) {
ossu20a4b3f2017-04-27 02:08:52 -0700211 channel_proxy->ResetSenderCongestionControlObjects();
212 stream->bandwidth_observer_.reset();
213 }
214
215 if (new_ids.transport_sequence_number != 0) {
216 channel_proxy->EnableSendTransportSequenceNumber(
217 new_ids.transport_sequence_number);
218 stream->transport_->send_side_cc()->EnablePeriodicAlrProbing(true);
219 stream->bandwidth_observer_.reset(stream->transport_->send_side_cc()
220 ->GetBitrateController()
221 ->CreateRtcpBandwidthObserver());
222 }
223
224 channel_proxy->RegisterSenderCongestionControlObjects(
225 stream->transport_, stream->bandwidth_observer_.get());
226 }
227
228 if (!ReconfigureSendCodec(stream, new_config)) {
229 LOG(LS_ERROR) << "Failed to set up send codec state.";
230 }
231
232 ReconfigureBitrateObserver(stream, new_config);
233 stream->config_ = new_config;
234}
235
solenberg3a941542015-11-16 07:34:50 -0800236void AudioSendStream::Start() {
elad.alond12a8e12017-03-23 11:04:48 -0700237 RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
minyue10cbb462016-11-07 09:29:22 -0800238 if (config_.min_bitrate_bps != -1 && config_.max_bitrate_bps != -1) {
Alex Narest78609d52017-10-20 10:37:47 +0200239 // Audio BWE is enabled.
240 transport_->packet_sender()->SetAccountForAudioPackets(true);
ossu20a4b3f2017-04-27 02:08:52 -0700241 ConfigureBitrateObserver(config_.min_bitrate_bps, config_.max_bitrate_bps);
mflodman86cc6ff2016-07-26 04:44:06 -0700242 }
243
Taylor Brandstetter1a018dc2016-03-08 12:37:39 -0800244 ScopedVoEInterface<VoEBase> base(voice_engine());
245 int error = base->StartSend(config_.voe_channel_id);
246 if (error != 0) {
247 LOG(LS_ERROR) << "AudioSendStream::Start failed with error: " << error;
248 }
solenberg3a941542015-11-16 07:34:50 -0800249}
250
251void AudioSendStream::Stop() {
elad.alond12a8e12017-03-23 11:04:48 -0700252 RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
ossu20a4b3f2017-04-27 02:08:52 -0700253 RemoveBitrateObserver();
perkj26091b12016-09-01 01:17:40 -0700254
Taylor Brandstetter1a018dc2016-03-08 12:37:39 -0800255 ScopedVoEInterface<VoEBase> base(voice_engine());
256 int error = base->StopSend(config_.voe_channel_id);
257 if (error != 0) {
258 LOG(LS_ERROR) << "AudioSendStream::Stop failed with error: " << error;
259 }
solenberg3a941542015-11-16 07:34:50 -0800260}
261
solenbergffbbcac2016-11-17 05:25:37 -0800262bool AudioSendStream::SendTelephoneEvent(int payload_type,
263 int payload_frequency, int event,
solenberg8842c3e2016-03-11 03:06:41 -0800264 int duration_ms) {
elad.alond12a8e12017-03-23 11:04:48 -0700265 RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
solenbergffbbcac2016-11-17 05:25:37 -0800266 return channel_proxy_->SetSendTelephoneEventPayloadType(payload_type,
267 payload_frequency) &&
Fredrik Solenbergb5727682015-12-04 15:22:19 +0100268 channel_proxy_->SendTelephoneEventOutband(event, duration_ms);
269}
270
solenberg94218532016-06-16 10:53:22 -0700271void AudioSendStream::SetMuted(bool muted) {
elad.alond12a8e12017-03-23 11:04:48 -0700272 RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
solenberg94218532016-06-16 10:53:22 -0700273 channel_proxy_->SetInputMute(muted);
274}
275
solenbergc7a8b082015-10-16 14:35:07 -0700276webrtc::AudioSendStream::Stats AudioSendStream::GetStats() const {
elad.alond12a8e12017-03-23 11:04:48 -0700277 RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
solenberg85a04962015-10-27 03:35:21 -0700278 webrtc::AudioSendStream::Stats stats;
279 stats.local_ssrc = config_.rtp.ssrc;
solenberg85a04962015-10-27 03:35:21 -0700280
solenberg358057b2015-11-27 10:46:42 -0800281 webrtc::CallStatistics call_stats = channel_proxy_->GetRTCPStatistics();
solenberg85a04962015-10-27 03:35:21 -0700282 stats.bytes_sent = call_stats.bytesSent;
283 stats.packets_sent = call_stats.packetsSent;
solenberg8b85de22015-11-16 09:48:04 -0800284 // RTT isn't known until a RTCP report is received. Until then, VoiceEngine
285 // returns 0 to indicate an error value.
286 if (call_stats.rttMs > 0) {
287 stats.rtt_ms = call_stats.rttMs;
288 }
289 // TODO(solenberg): [was ajm]: Re-enable this metric once we have a reliable
290 // implementation.
291 stats.aec_quality_min = -1;
solenberg85a04962015-10-27 03:35:21 -0700292
ossu20a4b3f2017-04-27 02:08:52 -0700293 if (config_.send_codec_spec) {
294 const auto& spec = *config_.send_codec_spec;
295 stats.codec_name = spec.format.name;
296 stats.codec_payload_type = rtc::Optional<int>(spec.payload_type);
solenberg85a04962015-10-27 03:35:21 -0700297
298 // Get data from the last remote RTCP report.
solenberg358057b2015-11-27 10:46:42 -0800299 for (const auto& block : channel_proxy_->GetRemoteRTCPReportBlocks()) {
solenberg8b85de22015-11-16 09:48:04 -0800300 // Lookup report for send ssrc only.
301 if (block.source_SSRC == stats.local_ssrc) {
302 stats.packets_lost = block.cumulative_num_packets_lost;
303 stats.fraction_lost = Q8ToFloat(block.fraction_lost);
304 stats.ext_seqnum = block.extended_highest_sequence_number;
ossu20a4b3f2017-04-27 02:08:52 -0700305 // Convert timestamps to milliseconds.
306 if (spec.format.clockrate_hz / 1000 > 0) {
solenberg8b85de22015-11-16 09:48:04 -0800307 stats.jitter_ms =
ossu20a4b3f2017-04-27 02:08:52 -0700308 block.interarrival_jitter / (spec.format.clockrate_hz / 1000);
solenberg85a04962015-10-27 03:35:21 -0700309 }
solenberg8b85de22015-11-16 09:48:04 -0800310 break;
solenberg85a04962015-10-27 03:35:21 -0700311 }
312 }
313 }
314
ivoc7aba0292016-11-14 04:52:06 -0800315 ScopedVoEInterface<VoEBase> base(voice_engine());
solenberg796b8f92017-03-01 17:02:23 -0800316 RTC_DCHECK(base->transmit_mixer());
317 stats.audio_level = base->transmit_mixer()->AudioLevelFullRange();
318 RTC_DCHECK_LE(0, stats.audio_level);
319
zsteine76bd3a2017-07-14 12:17:49 -0700320 stats.total_input_energy = base->transmit_mixer()->GetTotalInputEnergy();
321 stats.total_input_duration = base->transmit_mixer()->GetTotalInputDuration();
322
peaha9cc40b2017-06-29 08:32:09 -0700323 RTC_DCHECK(audio_state_->audio_processing());
324 auto audio_processing_stats =
325 audio_state_->audio_processing()->GetStatistics();
ivoc7aba0292016-11-14 04:52:06 -0800326 stats.echo_delay_median_ms = audio_processing_stats.delay_median;
327 stats.echo_delay_std_ms = audio_processing_stats.delay_standard_deviation;
328 stats.echo_return_loss = audio_processing_stats.echo_return_loss.instant();
329 stats.echo_return_loss_enhancement =
330 audio_processing_stats.echo_return_loss_enhancement.instant();
331 stats.residual_echo_likelihood =
332 audio_processing_stats.residual_echo_likelihood;
ivoc4e477a12017-01-15 08:29:46 -0800333 stats.residual_echo_likelihood_recent_max =
334 audio_processing_stats.residual_echo_likelihood_recent_max;
ivoc8c63a822016-10-21 04:10:03 -0700335
solenberg3a941542015-11-16 07:34:50 -0800336 internal::AudioState* audio_state =
337 static_cast<internal::AudioState*>(audio_state_.get());
solenberg566ef242015-11-06 15:34:49 -0800338 stats.typing_noise_detected = audio_state->typing_noise_detected();
ivoce1198e02017-09-08 08:13:19 -0700339 stats.ana_statistics = channel_proxy_->GetANAStatistics();
solenberg85a04962015-10-27 03:35:21 -0700340
341 return stats;
342}
343
pbos1ba8d392016-05-01 20:18:34 -0700344void AudioSendStream::SignalNetworkState(NetworkState state) {
elad.alond12a8e12017-03-23 11:04:48 -0700345 RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
pbos1ba8d392016-05-01 20:18:34 -0700346}
347
348bool AudioSendStream::DeliverRtcp(const uint8_t* packet, size_t length) {
349 // TODO(solenberg): Tests call this function on a network thread, libjingle
350 // calls on the worker thread. We should move towards always using a network
351 // thread. Then this check can be enabled.
elad.alond12a8e12017-03-23 11:04:48 -0700352 // RTC_DCHECK(!worker_thread_checker_.CalledOnValidThread());
pbos1ba8d392016-05-01 20:18:34 -0700353 return channel_proxy_->ReceivedRTCPPacket(packet, length);
354}
355
mflodman86cc6ff2016-07-26 04:44:06 -0700356uint32_t AudioSendStream::OnBitrateUpdated(uint32_t bitrate_bps,
357 uint8_t fraction_loss,
minyue78b4d562016-11-30 04:47:39 -0800358 int64_t rtt,
minyue93e45222017-05-18 14:32:41 -0700359 int64_t bwe_period_ms) {
stefanfca900a2017-04-10 03:53:00 -0700360 // A send stream may be allocated a bitrate of zero if the allocator decides
361 // to disable it. For now we ignore this decision and keep sending on min
362 // bitrate.
363 if (bitrate_bps == 0) {
364 bitrate_bps = config_.min_bitrate_bps;
365 }
mflodman86cc6ff2016-07-26 04:44:06 -0700366 RTC_DCHECK_GE(bitrate_bps,
minyue10cbb462016-11-07 09:29:22 -0800367 static_cast<uint32_t>(config_.min_bitrate_bps));
mflodman86cc6ff2016-07-26 04:44:06 -0700368 // The bitrate allocator might allocate an higher than max configured bitrate
369 // if there is room, to allow for, as example, extra FEC. Ignore that for now.
minyue10cbb462016-11-07 09:29:22 -0800370 const uint32_t max_bitrate_bps = config_.max_bitrate_bps;
mflodman86cc6ff2016-07-26 04:44:06 -0700371 if (bitrate_bps > max_bitrate_bps)
372 bitrate_bps = max_bitrate_bps;
373
minyue93e45222017-05-18 14:32:41 -0700374 channel_proxy_->SetBitrate(bitrate_bps, bwe_period_ms);
mflodman86cc6ff2016-07-26 04:44:06 -0700375
376 // The amount of audio protection is not exposed by the encoder, hence
377 // always returning 0.
378 return 0;
379}
380
elad.alond12a8e12017-03-23 11:04:48 -0700381void AudioSendStream::OnPacketAdded(uint32_t ssrc, uint16_t seq_num) {
382 RTC_DCHECK(pacer_thread_checker_.CalledOnValidThread());
383 // Only packets that belong to this stream are of interest.
384 if (ssrc == config_.rtp.ssrc) {
385 rtc::CritScope lock(&packet_loss_tracker_cs_);
eladalonedd6eea2017-05-25 00:15:35 -0700386 // TODO(eladalon): This function call could potentially reset the window,
elad.alond12a8e12017-03-23 11:04:48 -0700387 // setting both PLR and RPLR to unknown. Consider (during upcoming
388 // refactoring) passing an indication of such an event.
389 packet_loss_tracker_.OnPacketAdded(seq_num, rtc::TimeMillis());
390 }
391}
392
393void AudioSendStream::OnPacketFeedbackVector(
394 const std::vector<PacketFeedback>& packet_feedback_vector) {
eladalon3651fdd2017-08-24 07:26:25 -0700395 RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
elad.alond12a8e12017-03-23 11:04:48 -0700396 rtc::Optional<float> plr;
elad.alondadb4dc2017-03-23 15:29:50 -0700397 rtc::Optional<float> rplr;
elad.alond12a8e12017-03-23 11:04:48 -0700398 {
399 rtc::CritScope lock(&packet_loss_tracker_cs_);
400 packet_loss_tracker_.OnPacketFeedbackVector(packet_feedback_vector);
401 plr = packet_loss_tracker_.GetPacketLossRate();
elad.alondadb4dc2017-03-23 15:29:50 -0700402 rplr = packet_loss_tracker_.GetRecoverablePacketLossRate();
elad.alond12a8e12017-03-23 11:04:48 -0700403 }
eladalonedd6eea2017-05-25 00:15:35 -0700404 // TODO(eladalon): If R/PLR go back to unknown, no indication is given that
elad.alond12a8e12017-03-23 11:04:48 -0700405 // the previously sent value is no longer relevant. This will be taken care
406 // of with some refactoring which is now being done.
407 if (plr) {
408 channel_proxy_->OnTwccBasedUplinkPacketLossRate(*plr);
409 }
elad.alondadb4dc2017-03-23 15:29:50 -0700410 if (rplr) {
411 channel_proxy_->OnRecoverableUplinkPacketLossRate(*rplr);
412 }
elad.alond12a8e12017-03-23 11:04:48 -0700413}
414
michaelt79e05882016-11-08 02:50:09 -0800415void AudioSendStream::SetTransportOverhead(int transport_overhead_per_packet) {
elad.alond12a8e12017-03-23 11:04:48 -0700416 RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
nisseb8f9a322017-03-27 05:36:15 -0700417 transport_->send_side_cc()->SetTransportOverhead(
418 transport_overhead_per_packet);
michaelt79e05882016-11-08 02:50:09 -0800419 channel_proxy_->SetTransportOverhead(transport_overhead_per_packet);
420}
421
ossuc3d4b482017-05-23 06:07:11 -0700422RtpState AudioSendStream::GetRtpState() const {
423 return rtp_rtcp_module_->GetRtpState();
424}
425
sazac58f8c02017-07-19 00:39:19 -0700426const TimeInterval& AudioSendStream::GetActiveLifetime() const {
427 return active_lifetime_;
428}
429
solenberg3a941542015-11-16 07:34:50 -0800430VoiceEngine* AudioSendStream::voice_engine() const {
431 internal::AudioState* audio_state =
432 static_cast<internal::AudioState*>(audio_state_.get());
433 VoiceEngine* voice_engine = audio_state->voice_engine();
434 RTC_DCHECK(voice_engine);
435 return voice_engine;
solenbergc7a8b082015-10-16 14:35:07 -0700436}
minyue7a973442016-10-20 03:27:12 -0700437
438// Apply current codec settings to a single voe::Channel used for sending.
ossu20a4b3f2017-04-27 02:08:52 -0700439bool AudioSendStream::SetupSendCodec(AudioSendStream* stream,
440 const Config& new_config) {
441 RTC_DCHECK(new_config.send_codec_spec);
442 const auto& spec = *new_config.send_codec_spec;
minyue48368ad2017-05-10 04:06:11 -0700443
444 RTC_DCHECK(new_config.encoder_factory);
ossu20a4b3f2017-04-27 02:08:52 -0700445 std::unique_ptr<AudioEncoder> encoder =
446 new_config.encoder_factory->MakeAudioEncoder(spec.payload_type,
447 spec.format);
minyue7a973442016-10-20 03:27:12 -0700448
ossu20a4b3f2017-04-27 02:08:52 -0700449 if (!encoder) {
450 LOG(LS_ERROR) << "Unable to create encoder for " << spec.format;
451 return false;
452 }
453 // If a bitrate has been specified for the codec, use it over the
454 // codec's default.
455 if (spec.target_bitrate_bps) {
456 encoder->OnReceivedTargetAudioBitrate(*spec.target_bitrate_bps);
minyue7a973442016-10-20 03:27:12 -0700457 }
458
ossu20a4b3f2017-04-27 02:08:52 -0700459 // Enable ANA if configured (currently only used by Opus).
460 if (new_config.audio_network_adaptor_config) {
461 if (encoder->EnableAudioNetworkAdaptor(
462 *new_config.audio_network_adaptor_config, stream->event_log_)) {
minyue6b825df2016-10-31 04:08:32 -0700463 LOG(LS_INFO) << "Audio network adaptor enabled on SSRC "
ossu20a4b3f2017-04-27 02:08:52 -0700464 << new_config.rtp.ssrc;
465 } else {
466 RTC_NOTREACHED();
minyue6b825df2016-10-31 04:08:32 -0700467 }
minyue7a973442016-10-20 03:27:12 -0700468 }
469
ossu20a4b3f2017-04-27 02:08:52 -0700470 // Wrap the encoder in a an AudioEncoderCNG, if VAD is enabled.
471 if (spec.cng_payload_type) {
472 AudioEncoderCng::Config cng_config;
473 cng_config.num_channels = encoder->NumChannels();
474 cng_config.payload_type = *spec.cng_payload_type;
475 cng_config.speech_encoder = std::move(encoder);
476 cng_config.vad_mode = Vad::kVadNormal;
477 encoder.reset(new AudioEncoderCng(std::move(cng_config)));
ossu3b9ff382017-04-27 08:03:42 -0700478
479 stream->RegisterCngPayloadType(
480 *spec.cng_payload_type,
481 new_config.send_codec_spec->format.clockrate_hz);
minyue7a973442016-10-20 03:27:12 -0700482 }
ossu20a4b3f2017-04-27 02:08:52 -0700483
484 stream->channel_proxy_->SetEncoder(new_config.send_codec_spec->payload_type,
485 std::move(encoder));
minyue7a973442016-10-20 03:27:12 -0700486 return true;
487}
488
ossu20a4b3f2017-04-27 02:08:52 -0700489bool AudioSendStream::ReconfigureSendCodec(AudioSendStream* stream,
490 const Config& new_config) {
491 const auto& old_config = stream->config_;
minyue-webrtc8de18262017-07-26 14:18:40 +0200492
493 if (!new_config.send_codec_spec) {
494 // We cannot de-configure a send codec. So we will do nothing.
495 // By design, the send codec should have not been configured.
496 RTC_DCHECK(!old_config.send_codec_spec);
497 return true;
498 }
499
500 if (new_config.send_codec_spec == old_config.send_codec_spec &&
501 new_config.audio_network_adaptor_config ==
502 old_config.audio_network_adaptor_config) {
ossu20a4b3f2017-04-27 02:08:52 -0700503 return true;
504 }
505
506 // If we have no encoder, or the format or payload type's changed, create a
507 // new encoder.
508 if (!old_config.send_codec_spec ||
509 new_config.send_codec_spec->format !=
510 old_config.send_codec_spec->format ||
511 new_config.send_codec_spec->payload_type !=
512 old_config.send_codec_spec->payload_type) {
513 return SetupSendCodec(stream, new_config);
514 }
515
ossu20a4b3f2017-04-27 02:08:52 -0700516 const rtc::Optional<int>& new_target_bitrate_bps =
517 new_config.send_codec_spec->target_bitrate_bps;
518 // If a bitrate has been specified for the codec, use it over the
519 // codec's default.
520 if (new_target_bitrate_bps &&
521 new_target_bitrate_bps !=
522 old_config.send_codec_spec->target_bitrate_bps) {
523 CallEncoder(stream->channel_proxy_, [&](AudioEncoder* encoder) {
524 encoder->OnReceivedTargetAudioBitrate(*new_target_bitrate_bps);
525 });
526 }
527
528 ReconfigureANA(stream, new_config);
529 ReconfigureCNG(stream, new_config);
530
531 return true;
532}
533
534void AudioSendStream::ReconfigureANA(AudioSendStream* stream,
535 const Config& new_config) {
536 if (new_config.audio_network_adaptor_config ==
537 stream->config_.audio_network_adaptor_config) {
538 return;
539 }
540 if (new_config.audio_network_adaptor_config) {
541 CallEncoder(stream->channel_proxy_, [&](AudioEncoder* encoder) {
542 if (encoder->EnableAudioNetworkAdaptor(
543 *new_config.audio_network_adaptor_config, stream->event_log_)) {
544 LOG(LS_INFO) << "Audio network adaptor enabled on SSRC "
545 << new_config.rtp.ssrc;
546 } else {
547 RTC_NOTREACHED();
548 }
549 });
550 } else {
551 CallEncoder(stream->channel_proxy_, [&](AudioEncoder* encoder) {
552 encoder->DisableAudioNetworkAdaptor();
553 });
554 LOG(LS_INFO) << "Audio network adaptor disabled on SSRC "
555 << new_config.rtp.ssrc;
556 }
557}
558
559void AudioSendStream::ReconfigureCNG(AudioSendStream* stream,
560 const Config& new_config) {
561 if (new_config.send_codec_spec->cng_payload_type ==
562 stream->config_.send_codec_spec->cng_payload_type) {
563 return;
564 }
565
ossu3b9ff382017-04-27 08:03:42 -0700566 // Register the CNG payload type if it's been added, don't do anything if CNG
567 // is removed. Payload types must not be redefined.
568 if (new_config.send_codec_spec->cng_payload_type) {
569 stream->RegisterCngPayloadType(
570 *new_config.send_codec_spec->cng_payload_type,
571 new_config.send_codec_spec->format.clockrate_hz);
572 }
573
ossu20a4b3f2017-04-27 02:08:52 -0700574 // Wrap or unwrap the encoder in an AudioEncoderCNG.
575 stream->channel_proxy_->ModifyEncoder(
576 [&](std::unique_ptr<AudioEncoder>* encoder_ptr) {
577 std::unique_ptr<AudioEncoder> old_encoder(std::move(*encoder_ptr));
578 auto sub_encoders = old_encoder->ReclaimContainedEncoders();
579 if (!sub_encoders.empty()) {
580 // Replace enc with its sub encoder. We need to put the sub
581 // encoder in a temporary first, since otherwise the old value
582 // of enc would be destroyed before the new value got assigned,
583 // which would be bad since the new value is a part of the old
584 // value.
585 auto tmp = std::move(sub_encoders[0]);
586 old_encoder = std::move(tmp);
587 }
588 if (new_config.send_codec_spec->cng_payload_type) {
589 AudioEncoderCng::Config config;
590 config.speech_encoder = std::move(old_encoder);
591 config.num_channels = config.speech_encoder->NumChannels();
592 config.payload_type = *new_config.send_codec_spec->cng_payload_type;
593 config.vad_mode = Vad::kVadNormal;
594 encoder_ptr->reset(new AudioEncoderCng(std::move(config)));
595 } else {
596 *encoder_ptr = std::move(old_encoder);
597 }
598 });
599}
600
601void AudioSendStream::ReconfigureBitrateObserver(
602 AudioSendStream* stream,
603 const webrtc::AudioSendStream::Config& new_config) {
604 // Since the Config's default is for both of these to be -1, this test will
605 // allow us to configure the bitrate observer if the new config has bitrate
606 // limits set, but would only have us call RemoveBitrateObserver if we were
607 // previously configured with bitrate limits.
608 if (stream->config_.min_bitrate_bps == new_config.min_bitrate_bps &&
609 stream->config_.max_bitrate_bps == new_config.max_bitrate_bps) {
610 return;
611 }
612
613 if (new_config.min_bitrate_bps != -1 && new_config.max_bitrate_bps != -1) {
614 stream->ConfigureBitrateObserver(new_config.min_bitrate_bps,
615 new_config.max_bitrate_bps);
616 } else {
617 stream->RemoveBitrateObserver();
618 }
619}
620
621void AudioSendStream::ConfigureBitrateObserver(int min_bitrate_bps,
622 int max_bitrate_bps) {
623 RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
624 RTC_DCHECK_GE(max_bitrate_bps, min_bitrate_bps);
625 rtc::Event thread_sync_event(false /* manual_reset */, false);
626 worker_queue_->PostTask([&] {
627 // We may get a callback immediately as the observer is registered, so make
628 // sure the bitrate limits in config_ are up-to-date.
629 config_.min_bitrate_bps = min_bitrate_bps;
630 config_.max_bitrate_bps = max_bitrate_bps;
631 bitrate_allocator_->AddObserver(this, min_bitrate_bps, max_bitrate_bps, 0,
Alex Narestb3944f02017-10-13 14:56:18 +0200632 true, config_.track_id);
ossu20a4b3f2017-04-27 02:08:52 -0700633 thread_sync_event.Set();
634 });
635 thread_sync_event.Wait(rtc::Event::kForever);
636}
637
638void AudioSendStream::RemoveBitrateObserver() {
639 RTC_DCHECK(worker_thread_checker_.CalledOnValidThread());
640 rtc::Event thread_sync_event(false /* manual_reset */, false);
641 worker_queue_->PostTask([this, &thread_sync_event] {
642 bitrate_allocator_->RemoveObserver(this);
643 thread_sync_event.Set();
644 });
645 thread_sync_event.Wait(rtc::Event::kForever);
646}
647
ossu3b9ff382017-04-27 08:03:42 -0700648void AudioSendStream::RegisterCngPayloadType(int payload_type,
649 int clockrate_hz) {
ossu3b9ff382017-04-27 08:03:42 -0700650 const CodecInst codec = {payload_type, "CN", clockrate_hz, 0, 1, 0};
ossuc3d4b482017-05-23 06:07:11 -0700651 if (rtp_rtcp_module_->RegisterSendPayload(codec) != 0) {
652 rtp_rtcp_module_->DeRegisterSendPayload(codec.pltype);
653 if (rtp_rtcp_module_->RegisterSendPayload(codec) != 0) {
ossu3b9ff382017-04-27 08:03:42 -0700654 LOG(LS_ERROR) << "RegisterCngPayloadType() failed to register CN to "
655 "RTP/RTCP module";
656 }
657 }
658}
659
660
solenbergc7a8b082015-10-16 14:35:07 -0700661} // namespace internal
662} // namespace webrtc