External VNR speed improvement.
Improved visual quality with 3x times speed-up.
Change list:
1. Remove second chance filter in temporal denoising filter to mitigate trailing artifact.
2. Add swap buffer to save one whole-frame memcpy.
3. Do noise estimation on every N blocks.
4. Adopt a faster moving object detection algorithm (change the structure).
5. Refactor the for loops and PositionCheck().
6. Refactor the function ReduceFalseDetection (RFD).
7. Fix a bug in TrailingBlock() which causes a mismatch.
8. Change unit test to support swap buffer test.
9. Remove CopyMem8x8, use memcpy to copy U/V plane which can be optimized future.
10. Remove DenoiseMetrics.
Review URL: https://codereview.webrtc.org/1871853003
Cr-Commit-Position: refs/heads/master@{#12340}
diff --git a/webrtc/modules/video_processing/util/denoiser_filter.h b/webrtc/modules/video_processing/util/denoiser_filter.h
index f2c7570..1254a88 100644
--- a/webrtc/modules/video_processing/util/denoiser_filter.h
+++ b/webrtc/modules/video_processing/util/denoiser_filter.h
@@ -25,12 +25,6 @@
enum DenoiserDecision { COPY_BLOCK, FILTER_BLOCK };
enum CpuType { CPU_NEON, CPU_NOT_NEON };
-struct DenoiseMetrics {
- uint32_t var;
- uint32_t sad;
- uint8_t denoise;
- bool is_skin;
-};
class DenoiserFilter {
public:
@@ -43,10 +37,6 @@
int src_stride,
uint8_t* dst,
int dst_stride) = 0;
- virtual void CopyMem8x8(const uint8_t* src,
- int src_stride,
- uint8_t* dst,
- int dst_stride) = 0;
virtual uint32_t Variance16x8(const uint8_t* a,
int a_stride,
const uint8_t* b,
@@ -59,8 +49,7 @@
const uint8_t* sig,
int sig_stride,
uint8_t motion_magnitude,
- int increase_denoising,
- bool denoise_always) = 0;
+ int increase_denoising) = 0;
};
} // namespace webrtc
diff --git a/webrtc/modules/video_processing/util/denoiser_filter_c.cc b/webrtc/modules/video_processing/util/denoiser_filter_c.cc
index 8c84f49..1b3c0b7 100644
--- a/webrtc/modules/video_processing/util/denoiser_filter_c.cc
+++ b/webrtc/modules/video_processing/util/denoiser_filter_c.cc
@@ -25,17 +25,6 @@
}
}
-void DenoiserFilterC::CopyMem8x8(const uint8_t* src,
- int src_stride,
- uint8_t* dst,
- int dst_stride) {
- for (int i = 0; i < 8; i++) {
- memcpy(dst, src, 8);
- src += src_stride;
- dst += dst_stride;
- }
-}
-
uint32_t DenoiserFilterC::Variance16x8(const uint8_t* a,
int a_stride,
const uint8_t* b,
@@ -66,8 +55,7 @@
const uint8_t* sig,
int sig_stride,
uint8_t motion_magnitude,
- int increase_denoising,
- bool denoise_always) {
+ int increase_denoising) {
int sum_diff_thresh = 0;
int sum_diff = 0;
int adj_val[3] = {3, 4, 6};
@@ -137,60 +125,10 @@
sum_diff += col_sum[c];
}
- if (denoise_always)
- sum_diff_thresh = INT_MAX;
- else if (increase_denoising)
- sum_diff_thresh = kSumDiffThresholdHigh;
- else
- sum_diff_thresh = kSumDiffThreshold;
- if (abs(sum_diff) > sum_diff_thresh) {
- int delta = ((abs(sum_diff) - sum_diff_thresh) >> 8) + 1;
- // Only apply the adjustment for max delta up to 3.
- if (delta < 4) {
- sig -= sig_stride * 16;
- mc_running_avg_y -= mc_avg_y_stride * 16;
- running_avg_y -= avg_y_stride * 16;
- for (int r = 0; r < 16; ++r) {
- for (int c = 0; c < 16; ++c) {
- int diff = mc_running_avg_y[c] - sig[c];
- int adjustment = abs(diff);
- if (adjustment > delta)
- adjustment = delta;
- if (diff > 0) {
- // Bring denoised signal down.
- if (running_avg_y[c] - adjustment < 0)
- running_avg_y[c] = 0;
- else
- running_avg_y[c] = running_avg_y[c] - adjustment;
- col_sum[c] -= adjustment;
- } else if (diff < 0) {
- // Bring denoised signal up.
- if (running_avg_y[c] + adjustment > 255)
- running_avg_y[c] = 255;
- else
- running_avg_y[c] = running_avg_y[c] + adjustment;
- col_sum[c] += adjustment;
- }
- }
- sig += sig_stride;
- mc_running_avg_y += mc_avg_y_stride;
- running_avg_y += avg_y_stride;
- }
-
- sum_diff = 0;
- for (int c = 0; c < 16; ++c) {
- if (col_sum[c] >= 128) {
- col_sum[c] = 127;
- }
- sum_diff += col_sum[c];
- }
-
- if (abs(sum_diff) > sum_diff_thresh)
- return COPY_BLOCK;
- } else {
- return COPY_BLOCK;
- }
- }
+ sum_diff_thresh =
+ increase_denoising ? kSumDiffThresholdHigh : kSumDiffThreshold;
+ if (abs(sum_diff) > sum_diff_thresh)
+ return COPY_BLOCK;
return FILTER_BLOCK;
}
diff --git a/webrtc/modules/video_processing/util/denoiser_filter_c.h b/webrtc/modules/video_processing/util/denoiser_filter_c.h
index 3e52c3e..d8b6c5e 100644
--- a/webrtc/modules/video_processing/util/denoiser_filter_c.h
+++ b/webrtc/modules/video_processing/util/denoiser_filter_c.h
@@ -22,10 +22,6 @@
int src_stride,
uint8_t* dst,
int dst_stride) override;
- void CopyMem8x8(const uint8_t* src,
- int src_stride,
- uint8_t* dst,
- int dst_stride) override;
uint32_t Variance16x8(const uint8_t* a,
int a_stride,
const uint8_t* b,
@@ -38,8 +34,7 @@
const uint8_t* sig,
int sig_stride,
uint8_t motion_magnitude,
- int increase_denoising,
- bool denoise_always) override;
+ int increase_denoising) override;
};
} // namespace webrtc
diff --git a/webrtc/modules/video_processing/util/denoiser_filter_neon.cc b/webrtc/modules/video_processing/util/denoiser_filter_neon.cc
index 2920305..195b985 100644
--- a/webrtc/modules/video_processing/util/denoiser_filter_neon.cc
+++ b/webrtc/modules/video_processing/util/denoiser_filter_neon.cc
@@ -75,20 +75,6 @@
}
}
-void DenoiserFilterNEON::CopyMem8x8(const uint8_t* src,
- int src_stride,
- uint8_t* dst,
- int dst_stride) {
- uint8x8_t vtmp;
-
- for (int r = 0; r < 8; r++) {
- vtmp = vld1_u8(src);
- vst1_u8(dst, vtmp);
- src += src_stride;
- dst += dst_stride;
- }
-}
-
uint32_t DenoiserFilterNEON::Variance16x8(const uint8_t* a,
int a_stride,
const uint8_t* b,
@@ -106,8 +92,7 @@
const uint8_t* sig,
int sig_stride,
uint8_t motion_magnitude,
- int increase_denoising,
- bool denoise_always) {
+ int increase_denoising) {
// If motion_magnitude is small, making the denoiser more aggressive by
// increasing the adjustment for each level, level1 adjustment is
// increased, the deltas stay the same.
@@ -190,92 +175,13 @@
}
// Too much adjustments => copy block.
- {
- int64x1_t x = vqadd_s64(vget_high_s64(v_sum_diff_total),
- vget_low_s64(v_sum_diff_total));
- int sum_diff = vget_lane_s32(vabs_s32(vreinterpret_s32_s64(x)), 0);
- if (denoise_always)
- sum_diff_thresh = INT_MAX;
- else if (increase_denoising)
- sum_diff_thresh = kSumDiffThresholdHigh;
- else
- sum_diff_thresh = kSumDiffThreshold;
- if (sum_diff > sum_diff_thresh) {
- // Before returning to copy the block (i.e., apply no denoising),
- // checK if we can still apply some (weaker) temporal filtering to
- // this block, that would otherwise not be denoised at all. Simplest
- // is to apply an additional adjustment to running_avg_y to bring it
- // closer to sig. The adjustment is capped by a maximum delta, and
- // chosen such that in most cases the resulting sum_diff will be
- // within the accceptable range given by sum_diff_thresh.
-
- // The delta is set by the excess of absolute pixel diff over the
- // threshold.
- int delta = ((sum_diff - sum_diff_thresh) >> 8) + 1;
- // Only apply the adjustment for max delta up to 3.
- if (delta < 4) {
- const uint8x16_t k_delta = vmovq_n_u8(delta);
- sig -= sig_stride * 16;
- mc_running_avg_y -= mc_running_avg_y_stride * 16;
- running_avg_y -= running_avg_y_stride * 16;
- for (int r = 0; r < 16; ++r) {
- uint8x16_t v_running_avg_y = vld1q_u8(running_avg_y);
- const uint8x16_t v_sig = vld1q_u8(sig);
- const uint8x16_t v_mc_running_avg_y = vld1q_u8(mc_running_avg_y);
-
- // Calculate absolute difference and sign masks.
- const uint8x16_t v_abs_diff = vabdq_u8(v_sig, v_mc_running_avg_y);
- const uint8x16_t v_diff_pos_mask =
- vcltq_u8(v_sig, v_mc_running_avg_y);
- const uint8x16_t v_diff_neg_mask =
- vcgtq_u8(v_sig, v_mc_running_avg_y);
- // Clamp absolute difference to delta to get the adjustment.
- const uint8x16_t v_abs_adjustment = vminq_u8(v_abs_diff, (k_delta));
-
- const uint8x16_t v_pos_adjustment =
- vandq_u8(v_diff_pos_mask, v_abs_adjustment);
- const uint8x16_t v_neg_adjustment =
- vandq_u8(v_diff_neg_mask, v_abs_adjustment);
-
- v_running_avg_y = vqsubq_u8(v_running_avg_y, v_pos_adjustment);
- v_running_avg_y = vqaddq_u8(v_running_avg_y, v_neg_adjustment);
-
- // Store results.
- vst1q_u8(running_avg_y, v_running_avg_y);
-
- {
- const int8x16_t v_sum_diff =
- vqsubq_s8(vreinterpretq_s8_u8(v_neg_adjustment),
- vreinterpretq_s8_u8(v_pos_adjustment));
-
- const int16x8_t fe_dc_ba_98_76_54_32_10 = vpaddlq_s8(v_sum_diff);
- const int32x4_t fedc_ba98_7654_3210 =
- vpaddlq_s16(fe_dc_ba_98_76_54_32_10);
- const int64x2_t fedcba98_76543210 =
- vpaddlq_s32(fedc_ba98_7654_3210);
-
- v_sum_diff_total = vqaddq_s64(v_sum_diff_total, fedcba98_76543210);
- }
- // Update pointers for next iteration.
- sig += sig_stride;
- mc_running_avg_y += mc_running_avg_y_stride;
- running_avg_y += running_avg_y_stride;
- }
- {
- // Update the sum of all pixel differences of this MB.
- x = vqadd_s64(vget_high_s64(v_sum_diff_total),
- vget_low_s64(v_sum_diff_total));
- sum_diff = vget_lane_s32(vabs_s32(vreinterpret_s32_s64(x)), 0);
-
- if (sum_diff > sum_diff_thresh) {
- return COPY_BLOCK;
- }
- }
- } else {
- return COPY_BLOCK;
- }
- }
- }
+ int64x1_t x = vqadd_s64(vget_high_s64(v_sum_diff_total),
+ vget_low_s64(v_sum_diff_total));
+ int sum_diff = vget_lane_s32(vabs_s32(vreinterpret_s32_s64(x)), 0);
+ sum_diff_thresh =
+ increase_denoising ? kSumDiffThresholdHigh : kSumDiffThreshold;
+ if (sum_diff > sum_diff_thresh)
+ return COPY_BLOCK;
// Tell above level that block was filtered.
running_avg_y -= running_avg_y_stride * 16;
diff --git a/webrtc/modules/video_processing/util/denoiser_filter_neon.h b/webrtc/modules/video_processing/util/denoiser_filter_neon.h
index 2e3ea26..55850bd 100644
--- a/webrtc/modules/video_processing/util/denoiser_filter_neon.h
+++ b/webrtc/modules/video_processing/util/denoiser_filter_neon.h
@@ -22,10 +22,6 @@
int src_stride,
uint8_t* dst,
int dst_stride) override;
- void CopyMem8x8(const uint8_t* src,
- int src_stride,
- uint8_t* dst,
- int dst_stride) override;
uint32_t Variance16x8(const uint8_t* a,
int a_stride,
const uint8_t* b,
@@ -38,8 +34,7 @@
const uint8_t* sig,
int sig_stride,
uint8_t motion_magnitude,
- int increase_denoising,
- bool denoise_always) override;
+ int increase_denoising) override;
};
} // namespace webrtc
diff --git a/webrtc/modules/video_processing/util/denoiser_filter_sse2.cc b/webrtc/modules/video_processing/util/denoiser_filter_sse2.cc
index 614b6c9..0545a97 100644
--- a/webrtc/modules/video_processing/util/denoiser_filter_sse2.cc
+++ b/webrtc/modules/video_processing/util/denoiser_filter_sse2.cc
@@ -9,7 +9,6 @@
*/
#include <emmintrin.h>
-
#include "webrtc/modules/video_processing/util/denoiser_filter_sse2.h"
namespace webrtc {
@@ -110,18 +109,6 @@
}
}
-// TODO(jackychen): Optimize this function using SSE2.
-void DenoiserFilterSSE2::CopyMem8x8(const uint8_t* src,
- int src_stride,
- uint8_t* dst,
- int dst_stride) {
- for (int i = 0; i < 8; i++) {
- memcpy(dst, src, 8);
- src += src_stride;
- dst += dst_stride;
- }
-}
-
uint32_t DenoiserFilterSSE2::Variance16x8(const uint8_t* src,
int src_stride,
const uint8_t* ref,
@@ -139,8 +126,8 @@
const uint8_t* sig,
int sig_stride,
uint8_t motion_magnitude,
- int increase_denoising,
- bool denoise_always) {
+ int increase_denoising) {
+ DenoiserDecision decision = FILTER_BLOCK;
unsigned int sum_diff_thresh = 0;
int shift_inc =
(increase_denoising && motion_magnitude <= kMotionMagnitudeThreshold) ? 1
@@ -210,76 +197,13 @@
running_avg_y += avg_y_stride;
}
- {
- // Compute the sum of all pixel differences of this MB.
- unsigned int abs_sum_diff = AbsSumDiff16x1(acc_diff);
- if (denoise_always)
- sum_diff_thresh = INT_MAX;
- else if (increase_denoising)
- sum_diff_thresh = kSumDiffThresholdHigh;
- else
- sum_diff_thresh = kSumDiffThreshold;
- if (abs_sum_diff > sum_diff_thresh) {
- // Before returning to copy the block (i.e., apply no denoising),
- // check if we can still apply some (weaker) temporal filtering to
- // this block, that would otherwise not be denoised at all. Simplest
- // is to apply an additional adjustment to running_avg_y to bring it
- // closer to sig. The adjustment is capped by a maximum delta, and
- // chosen such that in most cases the resulting sum_diff will be
- // within the acceptable range given by sum_diff_thresh.
-
- // The delta is set by the excess of absolute pixel diff over the
- // threshold.
- int delta = ((abs_sum_diff - sum_diff_thresh) >> 8) + 1;
- // Only apply the adjustment for max delta up to 3.
- if (delta < 4) {
- const __m128i k_delta = _mm_set1_epi8(delta);
- sig -= sig_stride * 16;
- mc_running_avg_y -= mc_avg_y_stride * 16;
- running_avg_y -= avg_y_stride * 16;
- for (int r = 0; r < 16; ++r) {
- __m128i v_running_avg_y =
- _mm_loadu_si128(reinterpret_cast<__m128i*>(&running_avg_y[0]));
- // Calculate differences.
- const __m128i v_sig =
- _mm_loadu_si128(reinterpret_cast<const __m128i*>(&sig[0]));
- const __m128i v_mc_running_avg_y =
- _mm_loadu_si128(reinterpret_cast<__m128i*>(&mc_running_avg_y[0]));
- const __m128i pdiff = _mm_subs_epu8(v_mc_running_avg_y, v_sig);
- const __m128i ndiff = _mm_subs_epu8(v_sig, v_mc_running_avg_y);
- // Obtain the sign. FF if diff is negative.
- const __m128i diff_sign = _mm_cmpeq_epi8(pdiff, k_0);
- // Clamp absolute difference to delta to get the adjustment.
- const __m128i adj = _mm_min_epu8(_mm_or_si128(pdiff, ndiff), k_delta);
- // Restore the sign and get positive and negative adjustments.
- __m128i padj, nadj;
- padj = _mm_andnot_si128(diff_sign, adj);
- nadj = _mm_and_si128(diff_sign, adj);
- // Calculate filtered value.
- v_running_avg_y = _mm_subs_epu8(v_running_avg_y, padj);
- v_running_avg_y = _mm_adds_epu8(v_running_avg_y, nadj);
- _mm_storeu_si128(reinterpret_cast<__m128i*>(running_avg_y),
- v_running_avg_y);
-
- // Accumulate the adjustments.
- acc_diff = _mm_subs_epi8(acc_diff, padj);
- acc_diff = _mm_adds_epi8(acc_diff, nadj);
-
- // Update pointers for next iteration.
- sig += sig_stride;
- mc_running_avg_y += mc_avg_y_stride;
- running_avg_y += avg_y_stride;
- }
- abs_sum_diff = AbsSumDiff16x1(acc_diff);
- if (abs_sum_diff > sum_diff_thresh) {
- return COPY_BLOCK;
- }
- } else {
- return COPY_BLOCK;
- }
- }
- }
- return FILTER_BLOCK;
+ // Compute the sum of all pixel differences of this MB.
+ unsigned int abs_sum_diff = AbsSumDiff16x1(acc_diff);
+ sum_diff_thresh =
+ increase_denoising ? kSumDiffThresholdHigh : kSumDiffThreshold;
+ if (abs_sum_diff > sum_diff_thresh)
+ decision = COPY_BLOCK;
+ return decision;
}
} // namespace webrtc
diff --git a/webrtc/modules/video_processing/util/denoiser_filter_sse2.h b/webrtc/modules/video_processing/util/denoiser_filter_sse2.h
index 395fa10..731344c 100644
--- a/webrtc/modules/video_processing/util/denoiser_filter_sse2.h
+++ b/webrtc/modules/video_processing/util/denoiser_filter_sse2.h
@@ -22,10 +22,6 @@
int src_stride,
uint8_t* dst,
int dst_stride) override;
- void CopyMem8x8(const uint8_t* src,
- int src_stride,
- uint8_t* dst,
- int dst_stride) override;
uint32_t Variance16x8(const uint8_t* a,
int a_stride,
const uint8_t* b,
@@ -38,8 +34,7 @@
const uint8_t* sig,
int sig_stride,
uint8_t motion_magnitude,
- int increase_denoising,
- bool denoise_always) override;
+ int increase_denoising) override;
};
} // namespace webrtc
diff --git a/webrtc/modules/video_processing/util/noise_estimation.cc b/webrtc/modules/video_processing/util/noise_estimation.cc
index 87beac3..a0ae2c4 100644
--- a/webrtc/modules/video_processing/util/noise_estimation.cc
+++ b/webrtc/modules/video_processing/util/noise_estimation.cc
@@ -27,10 +27,10 @@
consec_low_var_[mb_index]++;
num_static_block_++;
if (consec_low_var_[mb_index] >= kConsecLowVarFrame &&
- (luma >> 8) < kAverageLumaMax && (luma >> 8) > kAverageLumaMin) {
+ (luma >> 6) < kAverageLumaMax && (luma >> 6) > kAverageLumaMin) {
// Normalized var by the average luma value, this gives more weight to
// darker blocks.
- int nor_var = var / (luma >> 12);
+ int nor_var = var / (luma >> 10);
noise_var_ +=
nor_var > kBlockSelectionVarMax ? kBlockSelectionVarMax : nor_var;
num_noisy_block_++;
@@ -46,25 +46,28 @@
// condition more reasonable.
// No enough samples implies the motion of the camera or too many moving
// objects in the frame.
- if (num_static_block_ < (0.65 * mb_cols_ * mb_rows_) || !num_noisy_block_) {
+ if (num_static_block_ <
+ (0.65 * mb_cols_ * mb_rows_ / NOISE_SUBSAMPLE_INTERVAL) ||
+ !num_noisy_block_) {
+#if DISPLAY
+ printf("Not enough samples. %d \n", num_static_block_);
+#endif
noise_var_ = 0;
noise_var_accum_ = 0;
- num_static_block_ = 0;
num_noisy_block_ = 0;
-#if DISPLAY
- printf("Not enough samples.\n");
-#endif
+ num_static_block_ = 0;
return;
} else {
+#if DISPLAY
+ printf("%d %d fraction = %.3f\n", num_static_block_,
+ mb_cols_ * mb_rows_ / NOISE_SUBSAMPLE_INTERVAL,
+ percent_static_block_);
+#endif
// Normalized by the number of noisy blocks.
noise_var_ /= num_noisy_block_;
// Get the percentage of static blocks.
- percent_static_block_ =
- static_cast<double>(num_static_block_) / (mb_cols_ * mb_rows_);
-#if DISPLAY
- printf("%d %d fraction = %.3f\n", num_static_block_, mb_cols_ * mb_rows_,
- percent_static_block_);
-#endif
+ percent_static_block_ = static_cast<double>(num_static_block_) /
+ (mb_cols_ * mb_rows_ / NOISE_SUBSAMPLE_INTERVAL);
num_noisy_block_ = 0;
num_static_block_ = 0;
}
@@ -75,12 +78,12 @@
} else {
noise_var_accum_ = (noise_var_accum_ * 15 + noise_var_) / 16;
}
- // Reset noise_var_ for the next frame.
- noise_var_ = 0;
#if DISPLAY
printf("noise_var_accum_ = %.1f, noise_var_ = %d.\n", noise_var_accum_,
noise_var_);
#endif
+ // Reset noise_var_ for the next frame.
+ noise_var_ = 0;
}
uint8_t NoiseEstimation::GetNoiseLevel() {
diff --git a/webrtc/modules/video_processing/util/noise_estimation.h b/webrtc/modules/video_processing/util/noise_estimation.h
index ca5cc23..24d44ca 100644
--- a/webrtc/modules/video_processing/util/noise_estimation.h
+++ b/webrtc/modules/video_processing/util/noise_estimation.h
@@ -18,7 +18,6 @@
namespace webrtc {
-#define EXPERIMENTAL 0
#define DISPLAY 0
const int kNoiseThreshold = 200;
@@ -28,11 +27,18 @@
const int kAverageLumaMax = 220;
const int kBlockSelectionVarMax = kNoiseThreshold << 1;
+// TODO(jackychen): To test different sampling strategy.
+// Collect noise data every NOISE_SUBSAMPLE_INTERVAL blocks.
+#define NOISE_SUBSAMPLE_INTERVAL 41
+
class NoiseEstimation {
public:
void Init(int width, int height, CpuType cpu_type);
+ // Collect noise data from one qualified block.
void GetNoise(int mb_index, uint32_t var, uint32_t luma);
+ // Reset the counter for consecutive low-var blocks.
void ResetConsecLowVar(int mb_index);
+ // Update noise level for current frame.
void UpdateNoiseLevel();
// 0: low noise, 1: high noise
uint8_t GetNoiseLevel();
@@ -42,13 +48,13 @@
int height_;
int mb_rows_;
int mb_cols_;
+ int num_noisy_block_;
+ int num_static_block_;
CpuType cpu_type_;
uint32_t noise_var_;
double noise_var_accum_;
- int num_noisy_block_;
- int num_static_block_;
double percent_static_block_;
- rtc::scoped_ptr<uint32_t[]> consec_low_var_;
+ std::unique_ptr<uint32_t[]> consec_low_var_;
};
} // namespace webrtc