webrtc/modules/video_coding/utility/frame_dropper.cc

/*
 *  Copyright (c) 2011 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "webrtc/modules/video_coding/utility/include/frame_dropper.h"

#include "webrtc/system_wrappers/include/trace.h"

namespace webrtc
{

const float kDefaultKeyFrameSizeAvgKBits = 0.9f;
const float kDefaultKeyFrameRatio = 0.99f;
const float kDefaultDropRatioAlpha = 0.9f;
const float kDefaultDropRatioMax = 0.96f;
const float kDefaultMaxTimeToDropFrames = 4.0f;  // In seconds.

FrameDropper::FrameDropper()
:
_keyFrameSizeAvgKbits(kDefaultKeyFrameSizeAvgKBits),
_keyFrameRatio(kDefaultKeyFrameRatio),
_dropRatio(kDefaultDropRatioAlpha, kDefaultDropRatioMax),
_enabled(true),
_max_time_drops(kDefaultMaxTimeToDropFrames)
{
    Reset();
}

FrameDropper::FrameDropper(float max_time_drops)
:
_keyFrameSizeAvgKbits(kDefaultKeyFrameSizeAvgKBits),
_keyFrameRatio(kDefaultKeyFrameRatio),
_dropRatio(kDefaultDropRatioAlpha, kDefaultDropRatioMax),
_enabled(true),
_max_time_drops(max_time_drops)
{
    Reset();
}

void
FrameDropper::Reset()
{
    _keyFrameRatio.Reset(0.99f);
    _keyFrameRatio.Apply(1.0f, 1.0f/300.0f); // 1 key frame every 10th second in 30 fps
    _keyFrameSizeAvgKbits.Reset(0.9f);
    _keyFrameCount = 0;
    _accumulator = 0.0f;
    _accumulatorMax = 150.0f; // assume 300 kb/s and 0.5 s window
    _targetBitRate = 300.0f;
    _incoming_frame_rate = 30;
    _keyFrameSpreadFrames = 0.5f * _incoming_frame_rate;
    _dropNext = false;
    _dropRatio.Reset(0.9f);
    _dropRatio.Apply(0.0f, 0.0f); // Initialize to 0
    _dropCount = 0;
    _windowSize = 0.5f;
    _wasBelowMax = true;
    _fastMode = false; // start with normal (non-aggressive) mode
    // Cap for the encoder buffer level/accumulator, in secs.
    _cap_buffer_size = 3.0f;
    // Cap on maximum amount of dropped frames between kept frames, in secs.
    _max_time_drops = 4.0f;
}

void
FrameDropper::Enable(bool enable)
{
    _enabled = enable;
}

void
FrameDropper::Fill(size_t frameSizeBytes, bool deltaFrame)
{
    if (!_enabled)
    {
        return;
    }
    float frameSizeKbits = 8.0f * static_cast<float>(frameSizeBytes) / 1000.0f;
    if (!deltaFrame && !_fastMode) // fast mode does not treat key-frames any different
    {
        _keyFrameSizeAvgKbits.Apply(1, frameSizeKbits);
        _keyFrameRatio.Apply(1.0, 1.0);
        if (frameSizeKbits > _keyFrameSizeAvgKbits.filtered())
        {
            // Remove the average key frame size since we
            // compensate for key frames when adding delta
            // frames.
            frameSizeKbits -= _keyFrameSizeAvgKbits.filtered();
        }
        else
        {
            // Shouldn't be negative, so zero is the lower bound.
            frameSizeKbits = 0;
        }
        if (_keyFrameRatio.filtered() > 1e-5 &&
            1 / _keyFrameRatio.filtered() < _keyFrameSpreadFrames)
        {
            // We are sending key frames more often than our upper bound for
            // how much we allow the key frame compensation to be spread
            // out in time. Therefor we must use the key frame ratio rather
            // than keyFrameSpreadFrames.
            _keyFrameCount =
                static_cast<int32_t>(1 / _keyFrameRatio.filtered() + 0.5);
        }
        else
        {
            // Compensate for the key frame the following frames
            _keyFrameCount = static_cast<int32_t>(_keyFrameSpreadFrames + 0.5);
        }
    }
    else
    {
        // Decrease the keyFrameRatio
        _keyFrameRatio.Apply(1.0, 0.0);
    }
    // Change the level of the accumulator (bucket)
    _accumulator += frameSizeKbits;
    CapAccumulator();
}

void
FrameDropper::Leak(uint32_t inputFrameRate)
{
    if (!_enabled)
    {
        return;
    }
    if (inputFrameRate < 1)
    {
        return;
    }
    if (_targetBitRate < 0.0f)
    {
        return;
    }
    _keyFrameSpreadFrames = 0.5f * inputFrameRate;
    // T is the expected bits per frame (target). If all frames were the same size,
    // we would get T bits per frame. Notice that T is also weighted to be able to
    // force a lower frame rate if wanted.
    float T = _targetBitRate / inputFrameRate;
    if (_keyFrameCount > 0)
    {
        // Perform the key frame compensation
        if (_keyFrameRatio.filtered() > 0 &&
            1 / _keyFrameRatio.filtered() < _keyFrameSpreadFrames)
        {
            T -= _keyFrameSizeAvgKbits.filtered() * _keyFrameRatio.filtered();
        }
        else
        {
            T -= _keyFrameSizeAvgKbits.filtered() / _keyFrameSpreadFrames;
        }
        _keyFrameCount--;
    }
    _accumulator -= T;
    if (_accumulator < 0.0f)
    {
        _accumulator = 0.0f;
    }
    UpdateRatio();
}

void
FrameDropper::UpdateNack(uint32_t nackBytes)
{
    if (!_enabled)
    {
        return;
    }
    _accumulator += static_cast<float>(nackBytes) * 8.0f / 1000.0f;
}

void
FrameDropper::FillBucket(float inKbits, float outKbits)
{
    _accumulator += (inKbits - outKbits);
}

void
FrameDropper::UpdateRatio()
{
    if (_accumulator > 1.3f * _accumulatorMax)
    {
        // Too far above accumulator max, react faster
        _dropRatio.UpdateBase(0.8f);
    }
    else
    {
        // Go back to normal reaction
        _dropRatio.UpdateBase(0.9f);
    }
    if (_accumulator > _accumulatorMax)
    {
        // We are above accumulator max, and should ideally
        // drop a frame. Increase the dropRatio and drop
        // the frame later.
        if (_wasBelowMax)
        {
            _dropNext = true;
        }
        if (_fastMode)
        {
            // always drop in aggressive mode
            _dropNext = true;
        }

        _dropRatio.Apply(1.0f, 1.0f);
        _dropRatio.UpdateBase(0.9f);
    }
    else
    {
        _dropRatio.Apply(1.0f, 0.0f);
    }
    _wasBelowMax = _accumulator < _accumulatorMax;
}

// This function signals when to drop frames to the caller. It makes use of the dropRatio
// to smooth out the drops over time.
bool
FrameDropper::DropFrame()
{
    if (!_enabled)
    {
        return false;
    }
    if (_dropNext)
    {
        _dropNext = false;
        _dropCount = 0;
    }

    if (_dropRatio.filtered() >= 0.5f) // Drops per keep
    {
        // limit is the number of frames we should drop between each kept frame
        // to keep our drop ratio. limit is positive in this case.
        float denom = 1.0f - _dropRatio.filtered();
        if (denom < 1e-5)
        {
            denom = (float)1e-5;
        }
        int32_t limit = static_cast<int32_t>(1.0f / denom - 1.0f + 0.5f);
        // Put a bound on the max amount of dropped frames between each kept
        // frame, in terms of frame rate and window size (secs).
        int max_limit = static_cast<int>(_incoming_frame_rate *
                                         _max_time_drops);
        if (limit > max_limit) {
          limit = max_limit;
        }
        if (_dropCount < 0)
        {
            // Reset the _dropCount since it was negative and should be positive.
            if (_dropRatio.filtered() > 0.4f)
            {
                _dropCount = -_dropCount;
            }
            else
            {
                _dropCount = 0;
            }
        }
        if (_dropCount < limit)
        {
            // As long we are below the limit we should drop frames.
            _dropCount++;
            return true;
        }
        else
        {
            // Only when we reset _dropCount a frame should be kept.
            _dropCount = 0;
            return false;
        }
    }
    else if (_dropRatio.filtered() > 0.0f &&
        _dropRatio.filtered() < 0.5f) // Keeps per drop
    {
        // limit is the number of frames we should keep between each drop
        // in order to keep the drop ratio. limit is negative in this case,
        // and the _dropCount is also negative.
        float denom = _dropRatio.filtered();
        if (denom < 1e-5)
        {
            denom = (float)1e-5;
        }
        int32_t limit = -static_cast<int32_t>(1.0f / denom - 1.0f + 0.5f);
        if (_dropCount > 0)
        {
            // Reset the _dropCount since we have a positive
            // _dropCount, and it should be negative.
            if (_dropRatio.filtered() < 0.6f)
            {
                _dropCount = -_dropCount;
            }
            else
            {
                _dropCount = 0;
            }
        }
        if (_dropCount > limit)
        {
            if (_dropCount == 0)
            {
                // Drop frames when we reset _dropCount.
                _dropCount--;
                return true;
            }
            else
            {
                // Keep frames as long as we haven't reached limit.
                _dropCount--;
                return false;
            }
        }
        else
        {
            _dropCount = 0;
            return false;
        }
    }
    _dropCount = 0;
    return false;

    // A simpler version, unfiltered and quicker
    //bool dropNext = _dropNext;
    //_dropNext = false;
    //return dropNext;
}

void
FrameDropper::SetRates(float bitRate, float incoming_frame_rate)
{
    // Bit rate of -1 means infinite bandwidth.
    _accumulatorMax = bitRate * _windowSize; // bitRate * windowSize (in seconds)
    if (_targetBitRate > 0.0f && bitRate < _targetBitRate && _accumulator > _accumulatorMax)
    {
        // Rescale the accumulator level if the accumulator max decreases
        _accumulator = bitRate / _targetBitRate * _accumulator;
    }
    _targetBitRate = bitRate;
    CapAccumulator();
    _incoming_frame_rate = incoming_frame_rate;
}

float
FrameDropper::ActualFrameRate(uint32_t inputFrameRate) const
{
    if (!_enabled)
    {
        return static_cast<float>(inputFrameRate);
    }
    return inputFrameRate * (1.0f - _dropRatio.filtered());
}

// Put a cap on the accumulator, i.e., don't let it grow beyond some level.
// This is a temporary fix for screencasting where very large frames from
// encoder will cause very slow response (too many frame drops).
void FrameDropper::CapAccumulator() {
  float max_accumulator = _targetBitRate * _cap_buffer_size;
  if (_accumulator > max_accumulator) {
    _accumulator = max_accumulator;
  }
}

}