webrtc/modules/audio_coding/neteq/payload_splitter.cc

/*
 *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
 *
 *  Use of this source code is governed by a BSD-style license
 *  that can be found in the LICENSE file in the root of the source
 *  tree. An additional intellectual property rights grant can be found
 *  in the file PATENTS.  All contributing project authors may
 *  be found in the AUTHORS file in the root of the source tree.
 */

#include "webrtc/modules/audio_coding/neteq/payload_splitter.h"

#include <assert.h>

#include "webrtc/base/checks.h"
#include "webrtc/base/logging.h"
#include "webrtc/modules/audio_coding/neteq/decoder_database.h"

namespace webrtc {

// The method loops through a list of packets {A, B, C, ...}. Each packet is
// split into its corresponding RED payloads, {A1, A2, ...}, which is
// temporarily held in the list |new_packets|.
// When the first packet in |packet_list| has been processed, the orignal packet
// is replaced by the new ones in |new_packets|, so that |packet_list| becomes:
// {A1, A2, ..., B, C, ...}. The method then continues with B, and C, until all
// the original packets have been replaced by their split payloads.
int PayloadSplitter::SplitRed(PacketList* packet_list) {
  int ret = kOK;
  PacketList::iterator it = packet_list->begin();
  while (it != packet_list->end()) {
    const Packet* red_packet = (*it);
    assert(!red_packet->payload.empty());
    const uint8_t* payload_ptr = red_packet->payload.data();

    // Read RED headers (according to RFC 2198):
    //
    //    0                   1                   2                   3
    //    0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
    //   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
    //   |F|   block PT  |  timestamp offset         |   block length    |
    //   +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
    // Last RED header:
    //    0 1 2 3 4 5 6 7
    //   +-+-+-+-+-+-+-+-+
    //   |0|   Block PT  |
    //   +-+-+-+-+-+-+-+-+

    struct RedHeader {
      uint8_t payload_type;
      uint32_t timestamp;
      size_t payload_length;
      bool primary;
    };

    std::vector<RedHeader> new_headers;
    bool last_block = false;
    size_t sum_length = 0;
    while (!last_block) {
      RedHeader new_header;
      // Check the F bit. If F == 0, this was the last block.
      last_block = ((*payload_ptr & 0x80) == 0);
      // Bits 1 through 7 are payload type.
      new_header.payload_type = payload_ptr[0] & 0x7F;
      if (last_block) {
        // No more header data to read.
        ++sum_length;  // Account for RED header size of 1 byte.
        new_header.timestamp = red_packet->header.timestamp;
        new_header.payload_length = red_packet->payload.size() - sum_length;
        new_header.primary = true;  // Last block is always primary.
        payload_ptr += 1;  // Advance to first payload byte.
      } else {
        // Bits 8 through 21 are timestamp offset.
        int timestamp_offset = (payload_ptr[1] << 6) +
            ((payload_ptr[2] & 0xFC) >> 2);
        new_header.timestamp = red_packet->header.timestamp - timestamp_offset;
        // Bits 22 through 31 are payload length.
        new_header.payload_length =
            ((payload_ptr[2] & 0x03) << 8) + payload_ptr[3];
        new_header.primary = false;
        payload_ptr += 4;  // Advance to next RED header.
      }
      sum_length += new_header.payload_length;
      sum_length += 4;  // Account for RED header size of 4 bytes.
      // Store in new list of packets.
      new_headers.push_back(new_header);
    }

    // Populate the new packets with payload data.
    // |payload_ptr| now points at the first payload byte.
    PacketList new_packets;  // An empty list to store the split packets in.
    for (const auto& new_header : new_headers) {
      size_t payload_length = new_header.payload_length;
      if (payload_ptr + payload_length >
          red_packet->payload.data() + red_packet->payload.size()) {
        // The block lengths in the RED headers do not match the overall packet
        // length. Something is corrupt. Discard this and the remaining
        // payloads from this packet.
        LOG(LS_WARNING) << "SplitRed length mismatch";
        ret = kRedLengthMismatch;
        break;
      }
      Packet* new_packet = new Packet;
      new_packet->header = red_packet->header;
      new_packet->header.timestamp = new_header.timestamp;
      new_packet->header.payloadType = new_header.payload_type;
      new_packet->primary = new_header.primary;
      new_packet->payload.SetData(payload_ptr, payload_length);
      new_packets.push_front(new_packet);
      payload_ptr += payload_length;
    }
    // Insert new packets into original list, before the element pointed to by
    // iterator |it|.
    packet_list->splice(it, new_packets, new_packets.begin(),
                        new_packets.end());
    // Delete old packet payload.
    delete (*it);
    // Remove |it| from the packet list. This operation effectively moves the
    // iterator |it| to the next packet in the list. Thus, we do not have to
    // increment it manually.
    it = packet_list->erase(it);
  }
  return ret;
}

int PayloadSplitter::SplitFec(PacketList* packet_list,
                              DecoderDatabase* decoder_database) {
  PacketList::iterator it = packet_list->begin();
  // Iterate through all packets in |packet_list|.
  while (it != packet_list->end()) {
    Packet* packet = (*it);  // Just to make the notation more intuitive.
    // Get codec type for this payload.
    uint8_t payload_type = packet->header.payloadType;
    const DecoderDatabase::DecoderInfo* info =
        decoder_database->GetDecoderInfo(payload_type);
    if (!info) {
      LOG(LS_WARNING) << "SplitFec unknown payload type";
      return kUnknownPayloadType;
    }

    // Not an FEC packet.
    AudioDecoder* decoder = decoder_database->GetDecoder(payload_type);
    // decoder should not return NULL, except for comfort noise payloads which
    // are handled separately.
    assert(decoder != NULL || decoder_database->IsComfortNoise(payload_type));
    if (!decoder ||
        !decoder->PacketHasFec(packet->payload.data(),
                               packet->payload.size())) {
      ++it;
      continue;
    }

    switch (info->codec_type) {
      case NetEqDecoder::kDecoderOpus:
      case NetEqDecoder::kDecoderOpus_2ch: {
        // The main payload of this packet should be decoded as a primary
        // payload, even if it comes as a secondary payload in a RED packet.
        packet->primary = true;

        Packet* new_packet = new Packet;
        new_packet->header = packet->header;
        int duration = decoder->PacketDurationRedundant(packet->payload.data(),
                                                        packet->payload.size());
        new_packet->header.timestamp -= duration;
        new_packet->payload.SetData(packet->payload);
        new_packet->primary = false;
        // Waiting time should not be set here.
        RTC_DCHECK(!packet->waiting_time);

        packet_list->insert(it, new_packet);
        break;
      }
      default: {
        LOG(LS_WARNING) << "SplitFec wrong payload type";
        return kFecSplitError;
      }
    }

    ++it;
  }
  return kOK;
}

int PayloadSplitter::CheckRedPayloads(PacketList* packet_list,
                                      const DecoderDatabase& decoder_database) {
  PacketList::iterator it = packet_list->begin();
  int main_payload_type = -1;
  int num_deleted_packets = 0;
  while (it != packet_list->end()) {
    uint8_t this_payload_type = (*it)->header.payloadType;
    if (!decoder_database.IsDtmf(this_payload_type) &&
        !decoder_database.IsComfortNoise(this_payload_type)) {
      if (main_payload_type == -1) {
        // This is the first packet in the list which is non-DTMF non-CNG.
        main_payload_type = this_payload_type;
      } else {
        if (this_payload_type != main_payload_type) {
          // We do not allow redundant payloads of a different type.
          // Discard this payload.
          delete (*it);
          // Remove |it| from the packet list. This operation effectively
          // moves the iterator |it| to the next packet in the list. Thus, we
          // do not have to increment it manually.
          it = packet_list->erase(it);
          ++num_deleted_packets;
          continue;
        }
      }
    }
    ++it;
  }
  return num_deleted_packets;
}

int PayloadSplitter::SplitAudio(PacketList* packet_list,
                                const DecoderDatabase& decoder_database) {
  PacketList::iterator it = packet_list->begin();
  // Iterate through all packets in |packet_list|.
  while (it != packet_list->end()) {
    Packet* packet = (*it);  // Just to make the notation more intuitive.
    // Get codec type for this payload.
    const DecoderDatabase::DecoderInfo* info =
        decoder_database.GetDecoderInfo(packet->header.payloadType);
    if (!info) {
      LOG(LS_WARNING) << "SplitAudio unknown payload type";
      return kUnknownPayloadType;
    }
    PacketList new_packets;
    switch (info->codec_type) {
      case NetEqDecoder::kDecoderPCMu:
      case NetEqDecoder::kDecoderPCMa: {
        // 8 bytes per ms; 8 timestamps per ms.
        SplitBySamples(packet, 8, 8, &new_packets);
        break;
      }
      case NetEqDecoder::kDecoderPCMu_2ch:
      case NetEqDecoder::kDecoderPCMa_2ch: {
        // 2 * 8 bytes per ms; 8 timestamps per ms.
        SplitBySamples(packet, 2 * 8, 8, &new_packets);
        break;
      }
      case NetEqDecoder::kDecoderG722: {
        // 8 bytes per ms; 16 timestamps per ms.
        SplitBySamples(packet, 8, 16, &new_packets);
        break;
      }
      case NetEqDecoder::kDecoderPCM16B: {
        // 16 bytes per ms; 8 timestamps per ms.
        SplitBySamples(packet, 16, 8, &new_packets);
        break;
      }
      case NetEqDecoder::kDecoderPCM16Bwb: {
        // 32 bytes per ms; 16 timestamps per ms.
        SplitBySamples(packet, 32, 16, &new_packets);
        break;
      }
      case NetEqDecoder::kDecoderPCM16Bswb32kHz: {
        // 64 bytes per ms; 32 timestamps per ms.
        SplitBySamples(packet, 64, 32, &new_packets);
        break;
      }
      case NetEqDecoder::kDecoderPCM16Bswb48kHz: {
        // 96 bytes per ms; 48 timestamps per ms.
        SplitBySamples(packet, 96, 48, &new_packets);
        break;
      }
      case NetEqDecoder::kDecoderPCM16B_2ch: {
        // 2 * 16 bytes per ms; 8 timestamps per ms.
        SplitBySamples(packet, 2 * 16, 8, &new_packets);
        break;
      }
      case NetEqDecoder::kDecoderPCM16Bwb_2ch: {
        // 2 * 32 bytes per ms; 16 timestamps per ms.
        SplitBySamples(packet, 2 * 32, 16, &new_packets);
        break;
      }
      case NetEqDecoder::kDecoderPCM16Bswb32kHz_2ch: {
        // 2 * 64 bytes per ms; 32 timestamps per ms.
        SplitBySamples(packet, 2 * 64, 32, &new_packets);
        break;
      }
      case NetEqDecoder::kDecoderPCM16Bswb48kHz_2ch: {
        // 2 * 96 bytes per ms; 48 timestamps per ms.
        SplitBySamples(packet, 2 * 96, 48, &new_packets);
        break;
      }
      case NetEqDecoder::kDecoderPCM16B_5ch: {
        // 5 * 16 bytes per ms; 8 timestamps per ms.
        SplitBySamples(packet, 5 * 16, 8, &new_packets);
        break;
      }
      case NetEqDecoder::kDecoderILBC: {
        size_t bytes_per_frame;
        int timestamps_per_frame;
        if (packet->payload.size() >= 950) {
          LOG(LS_WARNING) << "SplitAudio too large iLBC payload";
          return kTooLargePayload;
        }
        if (packet->payload.size() % 38 == 0) {
          // 20 ms frames.
          bytes_per_frame = 38;
          timestamps_per_frame = 160;
        } else if (packet->payload.size() % 50 == 0) {
          // 30 ms frames.
          bytes_per_frame = 50;
          timestamps_per_frame = 240;
        } else {
          LOG(LS_WARNING) << "SplitAudio invalid iLBC payload";
          return kFrameSplitError;
        }
        int ret = SplitByFrames(packet, bytes_per_frame, timestamps_per_frame,
                                &new_packets);
        if (ret < 0) {
          return ret;
        } else if (ret == kNoSplit) {
          // Do not split at all. Simply advance to the next packet in the list.
          ++it;
          // We do not have any new packets to insert, and should not delete the
          // old one. Skip the code after the switch case, and jump straight to
          // the next packet in the while loop.
          continue;
        }
        break;
      }
      default: {
        // Do not split at all. Simply advance to the next packet in the list.
        ++it;
        // We do not have any new packets to insert, and should not delete the
        // old one. Skip the code after the switch case, and jump straight to
        // the next packet in the while loop.
        continue;
      }
    }
    // Insert new packets into original list, before the element pointed to by
    // iterator |it|.
    packet_list->splice(it, new_packets, new_packets.begin(),
                        new_packets.end());
    // Delete old packet payload.
    delete (*it);
    // Remove |it| from the packet list. This operation effectively moves the
    // iterator |it| to the next packet in the list. Thus, we do not have to
    // increment it manually.
    it = packet_list->erase(it);
  }
  return kOK;
}

void PayloadSplitter::SplitBySamples(const Packet* packet,
                                     size_t bytes_per_ms,
                                     uint32_t timestamps_per_ms,
                                     PacketList* new_packets) {
  assert(packet);
  assert(new_packets);

  size_t split_size_bytes = packet->payload.size();

  // Find a "chunk size" >= 20 ms and < 40 ms.
  size_t min_chunk_size = bytes_per_ms * 20;
  // Reduce the split size by half as long as |split_size_bytes| is at least
  // twice the minimum chunk size (so that the resulting size is at least as
  // large as the minimum chunk size).
  while (split_size_bytes >= 2 * min_chunk_size) {
    split_size_bytes >>= 1;
  }
  uint32_t timestamps_per_chunk = static_cast<uint32_t>(
      split_size_bytes * timestamps_per_ms / bytes_per_ms);
  uint32_t timestamp = packet->header.timestamp;

  const uint8_t* payload_ptr = packet->payload.data();
  size_t len = packet->payload.size();
  while (len >= (2 * split_size_bytes)) {
    Packet* new_packet = new Packet;
    new_packet->header = packet->header;
    new_packet->header.timestamp = timestamp;
    timestamp += timestamps_per_chunk;
    new_packet->primary = packet->primary;
    new_packet->payload.SetData(payload_ptr, split_size_bytes);
    payload_ptr += split_size_bytes;
    new_packets->push_back(new_packet);
    len -= split_size_bytes;
  }

  if (len > 0) {
    Packet* new_packet = new Packet;
    new_packet->header = packet->header;
    new_packet->header.timestamp = timestamp;
    new_packet->primary = packet->primary;
    new_packet->payload.SetData(payload_ptr, len);
    new_packets->push_back(new_packet);
  }
}

int PayloadSplitter::SplitByFrames(const Packet* packet,
                                   size_t bytes_per_frame,
                                   uint32_t timestamps_per_frame,
                                   PacketList* new_packets) {
  if (packet->payload.size() % bytes_per_frame != 0) {
    LOG(LS_WARNING) << "SplitByFrames length mismatch";
    return kFrameSplitError;
  }

  if (packet->payload.size() == bytes_per_frame) {
    // Special case. Do not split the payload.
    return kNoSplit;
  }

  uint32_t timestamp = packet->header.timestamp;
  const uint8_t* payload_ptr = packet->payload.data();
  size_t len = packet->payload.size();
  while (len > 0) {
    assert(len >= bytes_per_frame);
    Packet* new_packet = new Packet;
    new_packet->header = packet->header;
    new_packet->header.timestamp = timestamp;
    timestamp += timestamps_per_frame;
    new_packet->primary = packet->primary;
    new_packet->payload.SetData(payload_ptr, bytes_per_frame);
    payload_ptr += bytes_per_frame;
    new_packets->push_back(new_packet);
    len -= bytes_per_frame;
  }
  return kOK;
}

}  // namespace webrtc