talk/app/webrtc/videosource.cc

/*
 * libjingle
 * Copyright 2012 Google Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are met:
 *
 *  1. Redistributions of source code must retain the above copyright notice,
 *     this list of conditions and the following disclaimer.
 *  2. Redistributions in binary form must reproduce the above copyright notice,
 *     this list of conditions and the following disclaimer in the documentation
 *     and/or other materials provided with the distribution.
 *  3. The name of the author may not be used to endorse or promote products
 *     derived from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED
 * WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
 * EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
 * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include "talk/app/webrtc/videosource.h"

#include <cstdlib>
#include <vector>

#include "talk/app/webrtc/mediaconstraintsinterface.h"
#include "talk/session/media/channelmanager.h"
#include "webrtc/base/arraysize.h"

using cricket::CaptureState;
using webrtc::MediaConstraintsInterface;
using webrtc::MediaSourceInterface;

namespace {

const double kRoundingTruncation = 0.0005;

enum {
  MSG_VIDEOCAPTURESTATECONNECT,
  MSG_VIDEOCAPTURESTATEDISCONNECT,
  MSG_VIDEOCAPTURESTATECHANGE,
};

// Default resolution. If no constraint is specified, this is the resolution we
// will use.
static const cricket::VideoFormatPod kDefaultFormat =
    {640, 480, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY};

// List of formats used if the camera doesn't support capability enumeration.
static const cricket::VideoFormatPod kVideoFormats[] = {
  {1920, 1080, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY},
  {1280, 720, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY},
  {960, 720, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY},
  {640, 360, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY},
  {640, 480, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY},
  {320, 240, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY},
  {320, 180, FPS_TO_INTERVAL(30), cricket::FOURCC_ANY}
};

MediaSourceInterface::SourceState
GetReadyState(cricket::CaptureState state) {
  switch (state) {
    case cricket::CS_STARTING:
      return MediaSourceInterface::kInitializing;
    case cricket::CS_RUNNING:
      return MediaSourceInterface::kLive;
    case cricket::CS_FAILED:
    case cricket::CS_NO_DEVICE:
    case cricket::CS_STOPPED:
      return MediaSourceInterface::kEnded;
    case cricket::CS_PAUSED:
      return MediaSourceInterface::kMuted;
    default:
      ASSERT(false && "GetReadyState unknown state");
  }
  return MediaSourceInterface::kEnded;
}

void SetUpperLimit(int new_limit, int* original_limit) {
  if (*original_limit < 0 || new_limit < *original_limit)
    *original_limit = new_limit;
}

// Updates |format_upper_limit| from |constraint|.
// If constraint.maxFoo is smaller than format_upper_limit.foo,
// set format_upper_limit.foo to constraint.maxFoo.
void SetUpperLimitFromConstraint(
    const MediaConstraintsInterface::Constraint& constraint,
    cricket::VideoFormat* format_upper_limit) {
  if (constraint.key == MediaConstraintsInterface::kMaxWidth) {
    int value = rtc::FromString<int>(constraint.value);
    SetUpperLimit(value, &(format_upper_limit->width));
  } else if (constraint.key == MediaConstraintsInterface::kMaxHeight) {
    int value = rtc::FromString<int>(constraint.value);
    SetUpperLimit(value, &(format_upper_limit->height));
  }
}

// Fills |format_out| with the max width and height allowed by |constraints|.
void FromConstraintsForScreencast(
    const MediaConstraintsInterface::Constraints& constraints,
    cricket::VideoFormat* format_out) {
  typedef MediaConstraintsInterface::Constraints::const_iterator
      ConstraintsIterator;

  cricket::VideoFormat upper_limit(-1, -1, 0, 0);
  for (ConstraintsIterator constraints_it = constraints.begin();
       constraints_it != constraints.end(); ++constraints_it)
    SetUpperLimitFromConstraint(*constraints_it, &upper_limit);

  if (upper_limit.width >= 0)
    format_out->width = upper_limit.width;
  if (upper_limit.height >= 0)
    format_out->height = upper_limit.height;
}

// Returns true if |constraint| is fulfilled. |format_out| can differ from
// |format_in| if the format is changed by the constraint. Ie - the frame rate
// can be changed by setting maxFrameRate.
bool NewFormatWithConstraints(
    const MediaConstraintsInterface::Constraint& constraint,
    const cricket::VideoFormat& format_in,
    bool mandatory,
    cricket::VideoFormat* format_out) {
  ASSERT(format_out != NULL);
  *format_out = format_in;

  if (constraint.key == MediaConstraintsInterface::kMinWidth) {
    int value = rtc::FromString<int>(constraint.value);
    return (value <= format_in.width);
  } else if (constraint.key == MediaConstraintsInterface::kMaxWidth) {
    int value = rtc::FromString<int>(constraint.value);
    return (value >= format_in.width);
  } else if (constraint.key == MediaConstraintsInterface::kMinHeight) {
    int value = rtc::FromString<int>(constraint.value);
    return (value <= format_in.height);
  } else if (constraint.key == MediaConstraintsInterface::kMaxHeight) {
    int value = rtc::FromString<int>(constraint.value);
    return (value >= format_in.height);
  } else if (constraint.key == MediaConstraintsInterface::kMinFrameRate) {
    int value = rtc::FromString<int>(constraint.value);
    return (value <= cricket::VideoFormat::IntervalToFps(format_in.interval));
  } else if (constraint.key == MediaConstraintsInterface::kMaxFrameRate) {
    int value = rtc::FromString<int>(constraint.value);
    if (value == 0) {
      if (mandatory) {
        // TODO(ronghuawu): Convert the constraint value to float when sub-1fps
        // is supported by the capturer.
        return false;
      } else {
        value = 1;
      }
    }
    if (value <= cricket::VideoFormat::IntervalToFps(format_in.interval))
      format_out->interval = cricket::VideoFormat::FpsToInterval(value);
    return true;
  } else if (constraint.key == MediaConstraintsInterface::kMinAspectRatio) {
    double value = rtc::FromString<double>(constraint.value);
    // The aspect ratio in |constraint.value| has been converted to a string and
    // back to a double, so it may have a rounding error.
    // E.g if the value 1/3 is converted to a string, the string will not have
    // infinite length.
    // We add a margin of 0.0005 which is high enough to detect the same aspect
    // ratio but small enough to avoid matching wrong aspect ratios.
    double ratio = static_cast<double>(format_in.width) / format_in.height;
    return  (value <= ratio + kRoundingTruncation);
  } else if (constraint.key == MediaConstraintsInterface::kMaxAspectRatio) {
    double value = rtc::FromString<double>(constraint.value);
    double ratio = static_cast<double>(format_in.width) / format_in.height;
    // Subtract 0.0005 to avoid rounding problems. Same as above.
    const double kRoundingTruncation = 0.0005;
    return  (value >= ratio - kRoundingTruncation);
  } else if (constraint.key == MediaConstraintsInterface::kNoiseReduction) {
    // These are actually options, not constraints, so they can be satisfied
    // regardless of the format.
    return true;
  }
  LOG(LS_WARNING) << "Found unknown MediaStream constraint. Name:"
      <<  constraint.key << " Value:" << constraint.value;
  return false;
}

// Removes cricket::VideoFormats from |formats| that don't meet |constraint|.
void FilterFormatsByConstraint(
    const MediaConstraintsInterface::Constraint& constraint,
    bool mandatory,
    std::vector<cricket::VideoFormat>* formats) {
  std::vector<cricket::VideoFormat>::iterator format_it =
      formats->begin();
  while (format_it != formats->end()) {
    // Modify the format_it to fulfill the constraint if possible.
    // Delete it otherwise.
    if (!NewFormatWithConstraints(constraint, (*format_it),
                                  mandatory, &(*format_it))) {
      format_it = formats->erase(format_it);
    } else {
      ++format_it;
    }
  }
}

// Returns a vector of cricket::VideoFormat that best match |constraints|.
std::vector<cricket::VideoFormat> FilterFormats(
    const MediaConstraintsInterface::Constraints& mandatory,
    const MediaConstraintsInterface::Constraints& optional,
    const std::vector<cricket::VideoFormat>& supported_formats) {
  typedef MediaConstraintsInterface::Constraints::const_iterator
      ConstraintsIterator;
  std::vector<cricket::VideoFormat> candidates = supported_formats;

  for (ConstraintsIterator constraints_it = mandatory.begin();
       constraints_it != mandatory.end(); ++constraints_it)
    FilterFormatsByConstraint(*constraints_it, true, &candidates);

  if (candidates.size() == 0)
    return candidates;

  // Ok - all mandatory checked and we still have a candidate.
  // Let's try filtering using the optional constraints.
  for (ConstraintsIterator  constraints_it = optional.begin();
       constraints_it != optional.end(); ++constraints_it) {
    std::vector<cricket::VideoFormat> current_candidates = candidates;
    FilterFormatsByConstraint(*constraints_it, false, &current_candidates);
    if (current_candidates.size() > 0) {
      candidates = current_candidates;
    }
  }

  // We have done as good as we can to filter the supported resolutions.
  return candidates;
}

// Find the format that best matches the default video size.
// Constraints are optional and since the performance of a video call
// might be bad due to bitrate limitations, CPU, and camera performance,
// it is better to select a resolution that is as close as possible to our
// default and still meets the contraints.
const cricket::VideoFormat& GetBestCaptureFormat(
    const std::vector<cricket::VideoFormat>& formats) {
  ASSERT(formats.size() > 0);

  int default_area = kDefaultFormat.width * kDefaultFormat.height;

  std::vector<cricket::VideoFormat>::const_iterator it = formats.begin();
  std::vector<cricket::VideoFormat>::const_iterator best_it = formats.begin();
  int best_diff_area = std::abs(default_area - it->width * it->height);
  int64_t best_diff_interval = kDefaultFormat.interval;
  for (; it != formats.end(); ++it) {
    int diff_area = std::abs(default_area - it->width * it->height);
    int64_t diff_interval = std::abs(kDefaultFormat.interval - it->interval);
    if (diff_area < best_diff_area ||
        (diff_area == best_diff_area && diff_interval < best_diff_interval)) {
      best_diff_area = diff_area;
      best_diff_interval = diff_interval;
      best_it = it;
    }
  }
  return *best_it;
}

// Set |option| to the highest-priority value of |key| in the constraints.
// Return false if the key is mandatory, and the value is invalid.
bool ExtractOption(const MediaConstraintsInterface* all_constraints,
                   const std::string& key,
                   rtc::Optional<bool>* option) {
  size_t mandatory = 0;
  bool value;
  if (FindConstraint(all_constraints, key, &value, &mandatory)) {
    *option = rtc::Optional<bool>(value);
    return true;
  }

  return mandatory == 0;
}

// Search |all_constraints| for known video options.  Apply all options that are
// found with valid values, and return false if any mandatory video option was
// found with an invalid value.
bool ExtractVideoOptions(const MediaConstraintsInterface* all_constraints,
                         cricket::VideoOptions* options) {
  bool all_valid = true;

  all_valid &= ExtractOption(all_constraints,
      MediaConstraintsInterface::kNoiseReduction,
      &(options->video_noise_reduction));

  return all_valid;
}

class FrameInputWrapper : public cricket::VideoRenderer {
 public:
  explicit FrameInputWrapper(cricket::VideoCapturer* capturer)
      : capturer_(capturer) {
    ASSERT(capturer_ != NULL);
  }

  virtual ~FrameInputWrapper() {}

  // VideoRenderer implementation.
  bool RenderFrame(const cricket::VideoFrame* frame) override {
    if (!capturer_->IsRunning()) {
      return true;
    }

    // This signal will be made on media engine render thread. The clients
    // of this signal should have no assumptions on what thread this signal
    // come from.
    capturer_->SignalVideoFrame(capturer_, frame);
    return true;
  }

 private:
  cricket::VideoCapturer* capturer_;

  RTC_DISALLOW_COPY_AND_ASSIGN(FrameInputWrapper);
};

}  // anonymous namespace

namespace webrtc {

rtc::scoped_refptr<VideoSource> VideoSource::Create(
    cricket::ChannelManager* channel_manager,
    cricket::VideoCapturer* capturer,
    const webrtc::MediaConstraintsInterface* constraints,
    bool remote) {
  ASSERT(channel_manager != NULL);
  ASSERT(capturer != NULL);
  rtc::scoped_refptr<VideoSource> source(new rtc::RefCountedObject<VideoSource>(
      channel_manager, capturer, remote));
  source->Initialize(constraints);
  return source;
}

VideoSource::VideoSource(cricket::ChannelManager* channel_manager,
                         cricket::VideoCapturer* capturer,
                         bool remote)
    : channel_manager_(channel_manager),
      video_capturer_(capturer),
      state_(kInitializing),
      remote_(remote) {
  channel_manager_->SignalVideoCaptureStateChange.connect(
      this, &VideoSource::OnStateChange);
}

VideoSource::~VideoSource() {
  channel_manager_->StopVideoCapture(video_capturer_.get(), format_);
  channel_manager_->SignalVideoCaptureStateChange.disconnect(this);
}

void VideoSource::Initialize(
    const webrtc::MediaConstraintsInterface* constraints) {

  std::vector<cricket::VideoFormat> formats =
      channel_manager_->GetSupportedFormats(video_capturer_.get());
  if (formats.empty()) {
    if (video_capturer_->IsScreencast()) {
      // The screen capturer can accept any resolution and we will derive the
      // format from the constraints if any.
      // Note that this only affects tab capturing, not desktop capturing,
      // since the desktop capturer does not respect the VideoFormat passed in.
      formats.push_back(cricket::VideoFormat(kDefaultFormat));
    } else {
      // The VideoCapturer implementation doesn't support capability
      // enumeration. We need to guess what the camera supports.
      for (int i = 0; i < arraysize(kVideoFormats); ++i) {
        formats.push_back(cricket::VideoFormat(kVideoFormats[i]));
      }
    }
  }

  if (constraints) {
    MediaConstraintsInterface::Constraints mandatory_constraints =
        constraints->GetMandatory();
    MediaConstraintsInterface::Constraints optional_constraints;
    optional_constraints = constraints->GetOptional();

    if (video_capturer_->IsScreencast()) {
      // Use the maxWidth and maxHeight allowed by constraints for screencast.
      FromConstraintsForScreencast(mandatory_constraints, &(formats[0]));
    }

    formats = FilterFormats(mandatory_constraints, optional_constraints,
                            formats);
  }

  if (formats.size() == 0) {
    LOG(LS_WARNING) << "Failed to find a suitable video format.";
    SetState(kEnded);
    return;
  }

  cricket::VideoOptions options;
  if (!ExtractVideoOptions(constraints, &options)) {
    LOG(LS_WARNING) << "Could not satisfy mandatory options.";
    SetState(kEnded);
    return;
  }
  options_.SetAll(options);

  format_ = GetBestCaptureFormat(formats);
  // Start the camera with our best guess.
  // TODO(perkj): Should we try again with another format it it turns out that
  // the camera doesn't produce frames with the correct format? Or will
  // cricket::VideCapturer be able to re-scale / crop to the requested
  // resolution?
  if (!channel_manager_->StartVideoCapture(video_capturer_.get(), format_)) {
    SetState(kEnded);
    return;
  }
  // Initialize hasn't succeeded until a successful state change has occurred.
}

cricket::VideoRenderer* VideoSource::FrameInput() {
  // Defer creation of frame_input_ until it's needed, e.g. the local video
  // sources will never need it.
  if (!frame_input_) {
    frame_input_.reset(new FrameInputWrapper(video_capturer_.get()));
  }
  return frame_input_.get();
}

void VideoSource::Stop() {
  channel_manager_->StopVideoCapture(video_capturer_.get(), format_);
}

void VideoSource::Restart() {
  if (!channel_manager_->StartVideoCapture(video_capturer_.get(), format_)) {
    SetState(kEnded);
    return;
  }
  for (auto* sink : sinks_) {
    channel_manager_->AddVideoSink(video_capturer_.get(), sink);
  }
}

void VideoSource::AddSink(
    rtc::VideoSinkInterface<cricket::VideoFrame>* output) {
  sinks_.push_back(output);
  channel_manager_->AddVideoSink(video_capturer_.get(), output);
}

void VideoSource::RemoveSink(
    rtc::VideoSinkInterface<cricket::VideoFrame>* output) {
  sinks_.remove(output);
  channel_manager_->RemoveVideoSink(video_capturer_.get(), output);
}

// OnStateChange listens to the ChannelManager::SignalVideoCaptureStateChange.
// This signal is triggered for all video capturers. Not only the one we are
// interested in.
void VideoSource::OnStateChange(cricket::VideoCapturer* capturer,
                                     cricket::CaptureState capture_state) {
  if (capturer == video_capturer_.get()) {
    SetState(GetReadyState(capture_state));
  }
}

void VideoSource::SetState(SourceState new_state) {
  // TODO(hbos): Temporarily disabled VERIFY due to webrtc:4776.
  // if (VERIFY(state_ != new_state)) {
  if (state_ != new_state) {
    state_ = new_state;
    FireOnChanged();
  }
}

}  // namespace webrtc