External denoiser based on noise estimation and moving object detection.
Improved the existing external denoiser in WebRTC: the filter strength
is adaptive based on the noise level of the whole frame and the moving
object detection result. The adaptive filter effectively removes the
artifacts in previous version, such as trailing and blockiness on moving
objects.
The external denoiser is off by default for now.
BUG=
Review URL: https://codereview.webrtc.org/1822333003
Cr-Commit-Position: refs/heads/master@{#12198}
diff --git a/webrtc/modules/video_processing/util/denoiser_filter.cc b/webrtc/modules/video_processing/util/denoiser_filter.cc
index a9c6f00..b111a0e 100644
--- a/webrtc/modules/video_processing/util/denoiser_filter.cc
+++ b/webrtc/modules/video_processing/util/denoiser_filter.cc
@@ -18,13 +18,16 @@
namespace webrtc {
const int kMotionMagnitudeThreshold = 8 * 3;
-const int kSumDiffThreshold = 16 * 16 * 2;
-const int kSumDiffThresholdHigh = 600;
+const int kSumDiffThreshold = 96;
+const int kSumDiffThresholdHigh = 448;
std::unique_ptr<DenoiserFilter> DenoiserFilter::Create(
- bool runtime_cpu_detection) {
+ bool runtime_cpu_detection,
+ CpuType* cpu_type) {
std::unique_ptr<DenoiserFilter> filter;
+ if (cpu_type != nullptr)
+ *cpu_type = CPU_NOT_NEON;
if (runtime_cpu_detection) {
// If we know the minimum architecture at compile time, avoid CPU detection.
#if defined(WEBRTC_ARCH_X86_FAMILY)
@@ -40,9 +43,13 @@
#endif
#elif defined(WEBRTC_HAS_NEON)
filter.reset(new DenoiserFilterNEON());
+ if (cpu_type != nullptr)
+ *cpu_type = CPU_NEON;
#elif defined(WEBRTC_DETECT_NEON)
if (WebRtc_GetCPUFeaturesARM() & kCPUFeatureNEON) {
filter.reset(new DenoiserFilterNEON());
+ if (cpu_type != nullptr)
+ *cpu_type = CPU_NEON;
} else {
filter.reset(new DenoiserFilterC());
}
diff --git a/webrtc/modules/video_processing/util/denoiser_filter.h b/webrtc/modules/video_processing/util/denoiser_filter.h
index 4074585..f2c7570 100644
--- a/webrtc/modules/video_processing/util/denoiser_filter.h
+++ b/webrtc/modules/video_processing/util/denoiser_filter.h
@@ -11,6 +11,7 @@
#ifndef WEBRTC_MODULES_VIDEO_PROCESSING_UTIL_DENOISER_FILTER_H_
#define WEBRTC_MODULES_VIDEO_PROCESSING_UTIL_DENOISER_FILTER_H_
+#include <climits>
#include <memory>
#include "webrtc/modules/include/module_common_types.h"
@@ -23,6 +24,7 @@
extern const int kSumDiffThresholdHigh;
enum DenoiserDecision { COPY_BLOCK, FILTER_BLOCK };
+enum CpuType { CPU_NEON, CPU_NOT_NEON };
struct DenoiseMetrics {
uint32_t var;
uint32_t sad;
@@ -32,7 +34,8 @@
class DenoiserFilter {
public:
- static std::unique_ptr<DenoiserFilter> Create(bool runtime_cpu_detection);
+ static std::unique_ptr<DenoiserFilter> Create(bool runtime_cpu_detection,
+ CpuType* cpu_type);
virtual ~DenoiserFilter() {}
@@ -56,7 +59,8 @@
const uint8_t* sig,
int sig_stride,
uint8_t motion_magnitude,
- int increase_denoising) = 0;
+ int increase_denoising,
+ bool denoise_always) = 0;
};
} // namespace webrtc
diff --git a/webrtc/modules/video_processing/util/denoiser_filter_c.cc b/webrtc/modules/video_processing/util/denoiser_filter_c.cc
index 6323980..8c84f49 100644
--- a/webrtc/modules/video_processing/util/denoiser_filter_c.cc
+++ b/webrtc/modules/video_processing/util/denoiser_filter_c.cc
@@ -66,7 +66,8 @@
const uint8_t* sig,
int sig_stride,
uint8_t motion_magnitude,
- int increase_denoising) {
+ int increase_denoising,
+ bool denoise_always) {
int sum_diff_thresh = 0;
int sum_diff = 0;
int adj_val[3] = {3, 4, 6};
@@ -136,9 +137,12 @@
sum_diff += col_sum[c];
}
- sum_diff_thresh = kSumDiffThreshold;
- if (increase_denoising)
+ if (denoise_always)
+ sum_diff_thresh = INT_MAX;
+ else if (increase_denoising)
sum_diff_thresh = kSumDiffThresholdHigh;
+ else
+ sum_diff_thresh = kSumDiffThreshold;
if (abs(sum_diff) > sum_diff_thresh) {
int delta = ((abs(sum_diff) - sum_diff_thresh) >> 8) + 1;
// Only apply the adjustment for max delta up to 3.
diff --git a/webrtc/modules/video_processing/util/denoiser_filter_c.h b/webrtc/modules/video_processing/util/denoiser_filter_c.h
index fe46ac3..3e52c3e 100644
--- a/webrtc/modules/video_processing/util/denoiser_filter_c.h
+++ b/webrtc/modules/video_processing/util/denoiser_filter_c.h
@@ -38,7 +38,8 @@
const uint8_t* sig,
int sig_stride,
uint8_t motion_magnitude,
- int increase_denoising) override;
+ int increase_denoising,
+ bool denoise_always) override;
};
} // namespace webrtc
diff --git a/webrtc/modules/video_processing/util/denoiser_filter_neon.cc b/webrtc/modules/video_processing/util/denoiser_filter_neon.cc
index b522bf0..2920305 100644
--- a/webrtc/modules/video_processing/util/denoiser_filter_neon.cc
+++ b/webrtc/modules/video_processing/util/denoiser_filter_neon.cc
@@ -106,13 +106,15 @@
const uint8_t* sig,
int sig_stride,
uint8_t motion_magnitude,
- int increase_denoising) {
+ int increase_denoising,
+ bool denoise_always) {
// If motion_magnitude is small, making the denoiser more aggressive by
// increasing the adjustment for each level, level1 adjustment is
// increased, the deltas stay the same.
int shift_inc =
(increase_denoising && motion_magnitude <= kMotionMagnitudeThreshold) ? 1
: 0;
+ int sum_diff_thresh = 0;
const uint8x16_t v_level1_adjustment = vmovq_n_u8(
(motion_magnitude <= kMotionMagnitudeThreshold) ? 4 + shift_inc : 3);
const uint8x16_t v_delta_level_1_and_2 = vdupq_n_u8(1);
@@ -192,10 +194,12 @@
int64x1_t x = vqadd_s64(vget_high_s64(v_sum_diff_total),
vget_low_s64(v_sum_diff_total));
int sum_diff = vget_lane_s32(vabs_s32(vreinterpret_s32_s64(x)), 0);
- int sum_diff_thresh = kSumDiffThreshold;
-
- if (increase_denoising)
+ if (denoise_always)
+ sum_diff_thresh = INT_MAX;
+ else if (increase_denoising)
sum_diff_thresh = kSumDiffThresholdHigh;
+ else
+ sum_diff_thresh = kSumDiffThreshold;
if (sum_diff > sum_diff_thresh) {
// Before returning to copy the block (i.e., apply no denoising),
// checK if we can still apply some (weaker) temporal filtering to
diff --git a/webrtc/modules/video_processing/util/denoiser_filter_neon.h b/webrtc/modules/video_processing/util/denoiser_filter_neon.h
index bc87ba7..2e3ea26 100644
--- a/webrtc/modules/video_processing/util/denoiser_filter_neon.h
+++ b/webrtc/modules/video_processing/util/denoiser_filter_neon.h
@@ -38,7 +38,8 @@
const uint8_t* sig,
int sig_stride,
uint8_t motion_magnitude,
- int increase_denoising) override;
+ int increase_denoising,
+ bool denoise_always) override;
};
} // namespace webrtc
diff --git a/webrtc/modules/video_processing/util/denoiser_filter_sse2.cc b/webrtc/modules/video_processing/util/denoiser_filter_sse2.cc
index 903d7b1..614b6c9 100644
--- a/webrtc/modules/video_processing/util/denoiser_filter_sse2.cc
+++ b/webrtc/modules/video_processing/util/denoiser_filter_sse2.cc
@@ -139,7 +139,9 @@
const uint8_t* sig,
int sig_stride,
uint8_t motion_magnitude,
- int increase_denoising) {
+ int increase_denoising,
+ bool denoise_always) {
+ unsigned int sum_diff_thresh = 0;
int shift_inc =
(increase_denoising && motion_magnitude <= kMotionMagnitudeThreshold) ? 1
: 0;
@@ -211,9 +213,12 @@
{
// Compute the sum of all pixel differences of this MB.
unsigned int abs_sum_diff = AbsSumDiff16x1(acc_diff);
- unsigned int sum_diff_thresh = kSumDiffThreshold;
- if (increase_denoising)
+ if (denoise_always)
+ sum_diff_thresh = INT_MAX;
+ else if (increase_denoising)
sum_diff_thresh = kSumDiffThresholdHigh;
+ else
+ sum_diff_thresh = kSumDiffThreshold;
if (abs_sum_diff > sum_diff_thresh) {
// Before returning to copy the block (i.e., apply no denoising),
// check if we can still apply some (weaker) temporal filtering to
diff --git a/webrtc/modules/video_processing/util/denoiser_filter_sse2.h b/webrtc/modules/video_processing/util/denoiser_filter_sse2.h
index 31d8510..395fa10 100644
--- a/webrtc/modules/video_processing/util/denoiser_filter_sse2.h
+++ b/webrtc/modules/video_processing/util/denoiser_filter_sse2.h
@@ -38,7 +38,8 @@
const uint8_t* sig,
int sig_stride,
uint8_t motion_magnitude,
- int increase_denoising) override;
+ int increase_denoising,
+ bool denoise_always) override;
};
} // namespace webrtc
diff --git a/webrtc/modules/video_processing/util/noise_estimation.cc b/webrtc/modules/video_processing/util/noise_estimation.cc
new file mode 100644
index 0000000..87beac3
--- /dev/null
+++ b/webrtc/modules/video_processing/util/noise_estimation.cc
@@ -0,0 +1,95 @@
+/*
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#include "webrtc/modules/video_processing/util/noise_estimation.h"
+
+namespace webrtc {
+
+void NoiseEstimation::Init(int width, int height, CpuType cpu_type) {
+ int mb_cols = width >> 4;
+ int mb_rows = height >> 4;
+ consec_low_var_.reset(new uint32_t[mb_cols * mb_rows]());
+ width_ = width;
+ height_ = height;
+ mb_cols_ = width_ >> 4;
+ mb_rows_ = height_ >> 4;
+ cpu_type_ = cpu_type;
+}
+
+void NoiseEstimation::GetNoise(int mb_index, uint32_t var, uint32_t luma) {
+ consec_low_var_[mb_index]++;
+ num_static_block_++;
+ if (consec_low_var_[mb_index] >= kConsecLowVarFrame &&
+ (luma >> 8) < kAverageLumaMax && (luma >> 8) > kAverageLumaMin) {
+ // Normalized var by the average luma value, this gives more weight to
+ // darker blocks.
+ int nor_var = var / (luma >> 12);
+ noise_var_ +=
+ nor_var > kBlockSelectionVarMax ? kBlockSelectionVarMax : nor_var;
+ num_noisy_block_++;
+ }
+}
+
+void NoiseEstimation::ResetConsecLowVar(int mb_index) {
+ consec_low_var_[mb_index] = 0;
+}
+
+void NoiseEstimation::UpdateNoiseLevel() {
+ // TODO(jackychen): Tune a threshold for numb_noisy_block > T to make the
+ // condition more reasonable.
+ // No enough samples implies the motion of the camera or too many moving
+ // objects in the frame.
+ if (num_static_block_ < (0.65 * mb_cols_ * mb_rows_) || !num_noisy_block_) {
+ noise_var_ = 0;
+ noise_var_accum_ = 0;
+ num_static_block_ = 0;
+ num_noisy_block_ = 0;
+#if DISPLAY
+ printf("Not enough samples.\n");
+#endif
+ return;
+ } else {
+ // Normalized by the number of noisy blocks.
+ noise_var_ /= num_noisy_block_;
+ // Get the percentage of static blocks.
+ percent_static_block_ =
+ static_cast<double>(num_static_block_) / (mb_cols_ * mb_rows_);
+#if DISPLAY
+ printf("%d %d fraction = %.3f\n", num_static_block_, mb_cols_ * mb_rows_,
+ percent_static_block_);
+#endif
+ num_noisy_block_ = 0;
+ num_static_block_ = 0;
+ }
+ // For the first frame just update the value with current noise_var_,
+ // otherwise, use the averaging window.
+ if (noise_var_accum_ == 0) {
+ noise_var_accum_ = noise_var_;
+ } else {
+ noise_var_accum_ = (noise_var_accum_ * 15 + noise_var_) / 16;
+ }
+ // Reset noise_var_ for the next frame.
+ noise_var_ = 0;
+#if DISPLAY
+ printf("noise_var_accum_ = %.1f, noise_var_ = %d.\n", noise_var_accum_,
+ noise_var_);
+#endif
+}
+
+uint8_t NoiseEstimation::GetNoiseLevel() {
+ int noise_thr = cpu_type_ ? kNoiseThreshold : kNoiseThresholdNeon;
+ UpdateNoiseLevel();
+ if (noise_var_accum_ > noise_thr) {
+ return 1;
+ }
+ return 0;
+}
+
+} // namespace webrtc
diff --git a/webrtc/modules/video_processing/util/noise_estimation.h b/webrtc/modules/video_processing/util/noise_estimation.h
new file mode 100644
index 0000000..ca5cc23
--- /dev/null
+++ b/webrtc/modules/video_processing/util/noise_estimation.h
@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+ *
+ * Use of this source code is governed by a BSD-style license
+ * that can be found in the LICENSE file in the root of the source
+ * tree. An additional intellectual property rights grant can be found
+ * in the file PATENTS. All contributing project authors may
+ * be found in the AUTHORS file in the root of the source tree.
+ */
+
+#ifndef WEBRTC_MODULES_VIDEO_PROCESSING_UTIL_NOISE_ESTIMATION_H_
+#define WEBRTC_MODULES_VIDEO_PROCESSING_UTIL_NOISE_ESTIMATION_H_
+
+#include "webrtc/base/scoped_ptr.h"
+#include "webrtc/modules/include/module_common_types.h"
+#include "webrtc/modules/video_processing/include/video_processing_defines.h"
+#include "webrtc/modules/video_processing/util/denoiser_filter.h"
+
+namespace webrtc {
+
+#define EXPERIMENTAL 0
+#define DISPLAY 0
+
+const int kNoiseThreshold = 200;
+const int kNoiseThresholdNeon = 70;
+const int kConsecLowVarFrame = 6;
+const int kAverageLumaMin = 20;
+const int kAverageLumaMax = 220;
+const int kBlockSelectionVarMax = kNoiseThreshold << 1;
+
+class NoiseEstimation {
+ public:
+ void Init(int width, int height, CpuType cpu_type);
+ void GetNoise(int mb_index, uint32_t var, uint32_t luma);
+ void ResetConsecLowVar(int mb_index);
+ void UpdateNoiseLevel();
+ // 0: low noise, 1: high noise
+ uint8_t GetNoiseLevel();
+
+ private:
+ int width_;
+ int height_;
+ int mb_rows_;
+ int mb_cols_;
+ CpuType cpu_type_;
+ uint32_t noise_var_;
+ double noise_var_accum_;
+ int num_noisy_block_;
+ int num_static_block_;
+ double percent_static_block_;
+ rtc::scoped_ptr<uint32_t[]> consec_low_var_;
+};
+
+} // namespace webrtc
+
+#endif // WEBRTC_MODULES_VIDEO_PROCESSING_UTIL_NOISE_ESTIMATION_H_