Introduce a stable playout delay mode for NetEq.
A packet arrival history is used to store the timing of incoming packets and tracks the earliest and latest packets by taking the difference between rtp timestamp and arrival time. The history is windowed to 2 seconds by default. The packet arrival history will replace the relative arrival delay tracker in a follow up cl.
The playout delay is estimated by taking the difference between the current playout timestamp and the earliest packet arrival in the history. This method works better when DTX is used compared to the buffer level filter that it replaces.
The threshold for acceleration is changed to be the maximum of the target delay and the maximum packet arrival delay in the history. This prevents any acceleration immediately after an underrun and gives some time to adapt the target delay to new network conditions.
The logic when to decode the next packet after a packet loss is also changed to do concealment for the full loss duration unless the delay is too high.
The new mode is default disabled and can be enabled using a field trial.
Bug: webrtc:13322,webrtc:13966
Change-Id: Idfa0020584591261475b9ca350cc7c6531de9911
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/259820
Reviewed-by: Minyue Li <minyue@webrtc.org>
Commit-Queue: Jakob Ivarsson‎ <jakobi@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#36899}
diff --git a/modules/audio_coding/neteq/decision_logic.h b/modules/audio_coding/neteq/decision_logic.h
index 22fb9f7..a0f590e 100644
--- a/modules/audio_coding/neteq/decision_logic.h
+++ b/modules/audio_coding/neteq/decision_logic.h
@@ -18,6 +18,7 @@
#include "api/neteq/tick_timer.h"
#include "modules/audio_coding/neteq/buffer_level_filter.h"
#include "modules/audio_coding/neteq/delay_manager.h"
+#include "modules/audio_coding/neteq/packet_arrival_history.h"
#include "rtc_base/experiments/field_trial_parser.h"
namespace webrtc {
@@ -25,10 +26,6 @@
// This is the class for the decision tree implementation.
class DecisionLogic : public NetEqController {
public:
- static const int kReinitAfterExpands = 100;
- static const int kMaxWaitForPacket = 10;
-
- // Constructor.
DecisionLogic(NetEqController::Config config);
DecisionLogic(NetEqController::Config config,
std::unique_ptr<DelayManager> delay_manager,
@@ -39,8 +36,8 @@
DecisionLogic(const DecisionLogic&) = delete;
DecisionLogic& operator=(const DecisionLogic&) = delete;
- // Resets object to a clean state.
- void Reset() override;
+ // Not used.
+ void Reset() override {}
// Resets parts of the state. Typically done when switching codecs.
void SoftReset() override;
@@ -73,7 +70,7 @@
// Adds `value` to `sample_memory_`.
void AddSampleMemory(int32_t value) override { sample_memory_ += value; }
- int TargetLevelMs() const override { return delay_manager_->TargetDelayMs(); }
+ int TargetLevelMs() const override;
absl::optional<int> PacketArrived(int fs_hz,
bool should_update_stats,
@@ -97,9 +94,7 @@
}
bool PeakFound() const override { return false; }
- int GetFilteredBufferLevel() const override {
- return buffer_level_filter_->filtered_current_level();
- }
+ int GetFilteredBufferLevel() const override;
// Accessors and mutators.
void set_sample_memory(int32_t value) override { sample_memory_ = value; }
@@ -124,30 +119,20 @@
// Returns the operation given that the next available packet is a comfort
// noise payload (RFC 3389 only, not codec-internal).
- virtual NetEq::Operation CngOperation(NetEq::Mode prev_mode,
- uint32_t target_timestamp,
- uint32_t available_timestamp,
- size_t generated_noise_samples);
+ virtual NetEq::Operation CngOperation(NetEqController::NetEqStatus status);
// Returns the operation given that no packets are available (except maybe
// a DTMF event, flagged by setting `play_dtmf` true).
- virtual NetEq::Operation NoPacket(bool play_dtmf);
+ virtual NetEq::Operation NoPacket(NetEqController::NetEqStatus status);
// Returns the operation to do given that the expected packet is available.
- virtual NetEq::Operation ExpectedPacketAvailable(NetEq::Mode prev_mode,
- bool play_dtmf);
+ virtual NetEq::Operation ExpectedPacketAvailable(
+ NetEqController::NetEqStatus status);
// Returns the operation to do given that the expected packet is not
// available, but a packet further into the future is at hand.
virtual NetEq::Operation FuturePacketAvailable(
- size_t decoder_frame_length,
- NetEq::Mode prev_mode,
- uint32_t target_timestamp,
- uint32_t available_timestamp,
- bool play_dtmf,
- size_t generated_noise_samples,
- size_t span_samples_in_packet_buffer,
- size_t num_packets_in_packet_buffer);
+ NetEqController::NetEqStatus status);
// Checks if enough time has elapsed since the last successful timescale
// operation was done (i.e., accelerate or preemptive expand).
@@ -167,13 +152,34 @@
// conveyed in `timestamp_leap`.
bool PacketTooEarly(uint32_t timestamp_leap) const;
- // Checks if num_consecutive_expands_ >= kMaxWaitForPacket.
bool MaxWaitForPacket() const;
+ bool ShouldContinueExpand(NetEqController::NetEqStatus status) const;
+
+ int GetNextPacketDelayMs(NetEqController::NetEqStatus status) const;
+ int GetPlayoutDelayMs(NetEqController::NetEqStatus status) const;
+
+ int LowThreshold() const;
+ int HighThreshold() const;
+ int LowThresholdCng() const;
+ int HighThresholdCng() const;
+
+ // Runtime configurable options through field trial
+ // WebRTC-Audio-NetEqDecisionLogicConfig.
+ struct Config {
+ Config();
+
+ bool enable_stable_playout_delay = false;
+ int reinit_after_expands = 100;
+ int deceleration_target_level_offset_ms = 85;
+ int packet_history_size_ms = 2000;
+ };
+ Config config_;
std::unique_ptr<DelayManager> delay_manager_;
std::unique_ptr<BufferLevelFilter> buffer_level_filter_;
+ PacketArrivalHistory packet_arrival_history_;
const TickTimer* tick_timer_;
- int sample_rate_;
+ int sample_rate_khz_;
size_t output_size_samples_;
CngState cng_state_ = kCngOff; // Remember if comfort noise is interrupted by
// other event (e.g., DTMF).
@@ -187,7 +193,7 @@
int time_stretched_cn_samples_ = 0;
bool last_pack_cng_or_dtmf_ = true;
bool buffer_flush_ = false;
- FieldTrialConstrained<int> target_level_window_ms_;
+ int last_playout_delay_ms_ = 0;
};
} // namespace webrtc