Add plumbing of RtpPacketInfos to each AudioFrame as input for SourceTracker.
This change adds the plumbing of RtpPacketInfo from ChannelReceive::OnRtpPacket() to ChannelReceive::GetAudioFrameWithInfo() for audio. It is a step towards replacing the non-spec compliant ContributingSources that updates itself at packet-receive time, with the spec-compliant SourceTracker that will update itself at frame-delivery-to-track time.
Bug: webrtc:10668
Change-Id: I03385d6865bbc7bfbef7634f88de820a934f787a
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/139890
Reviewed-by: Stefan Holmer <stefan@webrtc.org>
Reviewed-by: Minyue Li <minyue@webrtc.org>
Commit-Queue: Chen Xing <chxg@google.com>
Cr-Commit-Position: refs/heads/master@{#28434}
diff --git a/modules/audio_coding/neteq/neteq_impl.cc b/modules/audio_coding/neteq/neteq_impl.cc
index 6f36fb1..62184b0 100644
--- a/modules/audio_coding/neteq/neteq_impl.cc
+++ b/modules/audio_coding/neteq/neteq_impl.cc
@@ -15,6 +15,7 @@
#include <cstdint>
#include <cstring>
#include <list>
+#include <map>
#include <utility>
#include <vector>
@@ -52,13 +53,16 @@
#include "rtc_base/sanitizer.h"
#include "rtc_base/strings/audio_format_to_string.h"
#include "rtc_base/trace_event.h"
+#include "system_wrappers/include/clock.h"
namespace webrtc {
NetEqImpl::Dependencies::Dependencies(
const NetEq::Config& config,
+ Clock* clock,
const rtc::scoped_refptr<AudioDecoderFactory>& decoder_factory)
- : tick_timer(new TickTimer),
+ : clock(clock),
+ tick_timer(new TickTimer),
stats(new StatisticsCalculator),
buffer_level_filter(new BufferLevelFilter),
decoder_database(
@@ -86,7 +90,8 @@
NetEqImpl::NetEqImpl(const NetEq::Config& config,
Dependencies&& deps,
bool create_components)
- : tick_timer_(std::move(deps.tick_timer)),
+ : clock_(deps.clock),
+ tick_timer_(std::move(deps.tick_timer)),
buffer_level_filter_(std::move(deps.buffer_level_filter)),
decoder_database_(std::move(deps.decoder_database)),
delay_manager_(std::move(deps.delay_manager)),
@@ -468,17 +473,20 @@
RTC_LOG_F(LS_ERROR) << "payload is empty";
return kInvalidPointer;
}
+
+ int64_t receive_time_ms = clock_->TimeInMilliseconds();
stats_->ReceivedPacket();
PacketList packet_list;
// Insert packet in a packet list.
- packet_list.push_back([&rtp_header, &payload] {
+ packet_list.push_back([&rtp_header, &payload, &receive_time_ms] {
// Convert to Packet.
Packet packet;
packet.payload_type = rtp_header.payloadType;
packet.sequence_number = rtp_header.sequenceNumber;
packet.timestamp = rtp_header.timestamp;
packet.payload.SetData(payload.data(), payload.size());
+ packet.packet_info = RtpPacketInfo(rtp_header, receive_time_ms);
// Waiting time will be set upon inserting the packet in the buffer.
RTC_DCHECK(!packet.waiting_time);
return packet;
@@ -611,6 +619,7 @@
const auto sequence_number = packet.sequence_number;
const auto payload_type = packet.payload_type;
const Packet::Priority original_priority = packet.priority;
+ const auto& packet_info = packet.packet_info;
auto packet_from_result = [&](AudioDecoder::ParseResult& result) {
Packet new_packet;
new_packet.sequence_number = sequence_number;
@@ -618,6 +627,7 @@
new_packet.timestamp = result.timestamp;
new_packet.priority.codec_level = result.priority;
new_packet.priority.red_level = original_priority.red_level;
+ new_packet.packet_info = packet_info;
new_packet.frame = std::move(result.frame);
return new_packet;
};
@@ -879,7 +889,16 @@
comfort_noise_->Reset();
}
- // Copy from |algorithm_buffer| to |sync_buffer_|.
+ // We treat it as if all packets referenced to by |last_decoded_packet_infos_|
+ // were mashed together when creating the samples in |algorithm_buffer_|.
+ RtpPacketInfos packet_infos(std::move(last_decoded_packet_infos_));
+ last_decoded_packet_infos_.clear();
+
+ // Copy samples from |algorithm_buffer_| to |sync_buffer_|.
+ //
+ // TODO(bugs.webrtc.org/10757):
+ // We would in the future also like to pass |packet_infos| so that we can do
+ // sample-perfect tracking of that information across |sync_buffer_|.
sync_buffer_->PushBack(*algorithm_buffer_);
// Extract data from |sync_buffer_| to |output|.
@@ -897,6 +916,13 @@
sync_buffer_->GetNextAudioInterleaved(num_output_samples_per_channel,
audio_frame);
audio_frame->sample_rate_hz_ = fs_hz_;
+ // TODO(bugs.webrtc.org/10757):
+ // We don't have the ability to properly track individual packets once their
+ // audio samples have entered |sync_buffer_|. So for now, treat it as if
+ // |packet_infos| from packets decoded by the current |GetAudioInternal()|
+ // call were all consumed assembling the current audio frame and the current
+ // audio frame only.
+ audio_frame->packet_infos_ = std::move(packet_infos);
if (sync_buffer_->FutureLength() < expand_->overlap_length()) {
// The sync buffer should always contain |overlap_length| samples, but now
// too many samples have been extracted. Reinstall the |overlap_length|
@@ -1392,6 +1418,7 @@
int* decoded_length,
AudioDecoder::SpeechType* speech_type) {
RTC_DCHECK(last_decoded_timestamps_.empty());
+ RTC_DCHECK(last_decoded_packet_infos_.empty());
// Do decoding.
while (!packet_list->empty() && !decoder_database_->IsComfortNoise(
@@ -1409,6 +1436,8 @@
rtc::ArrayView<int16_t>(&decoded_buffer_[*decoded_length],
decoded_buffer_length_ - *decoded_length));
last_decoded_timestamps_.push_back(packet_list->front().timestamp);
+ last_decoded_packet_infos_.push_back(
+ std::move(packet_list->front().packet_info));
packet_list->pop_front();
if (opt_result) {
const auto& result = *opt_result;
@@ -1424,6 +1453,7 @@
// TODO(ossu): What to put here?
RTC_LOG(LS_WARNING) << "Decode error";
*decoded_length = -1;
+ last_decoded_packet_infos_.clear();
packet_list->clear();
break;
}