commit 81cde45e589f3ba341c06837fb1c0d775c0760ba
author Kuang-che Wu <kcwu@chromium.org> Mon Apr 08 16:56:51 2019 +0800
committer chrome-bot <chrome-bot@chromium.org> Tue Apr 09 01:27:05 2019 -0700
tree 575570b57145822cc1db518cce1ac3103453ec11
parent 356ecb9c2920905a55fec2b078e79d2f50755d57

bisect-kit: refactor test value handling

Currently, eval script converts performance score into boolean values.
Bisector's main algorithm has no knowledge of the actual value.

This CL moves the logic from eval script to bisector (from
eval_cros_autotest.py to strategy.py). With this refactoring, bisector
can tweak the algorithm later specific for numeric value bisection
(instead of boolean value), for example,
 - recompute initial values
 - even more, bisect according to the distribution of values

BUG=chromium:881238
TEST=run diagnose_cros_autotest.py manually

Change-Id: I144b91c46be50b72cd01cb48c12adcd8660b67ec
Reviewed-on: https://chromium-review.googlesource.com/1556981
Commit-Ready: Kuang-che Wu <kcwu@chromium.org>
Tested-by: Kuang-che Wu <kcwu@chromium.org>
Reviewed-by: Chung-yih Wang <cywang@chromium.org>
Reviewed-by: Chi-Ngai Wan <cnwan@google.com>

eval_cros_autotest.py

diff --git a/eval_cros_autotest.py b/eval_cros_autotest.py
index 5eab347..a924c0f 100755
--- a/eval_cros_autotest.py
+++ b/eval_cros_autotest.py
@@ -89,14 +89,6 @@
       help=
       'Metric name of performance test; example: "cheets_SystemRawImageSize"')
   group.add_argument(
-      '--old_value',
-      type=float,
-      help='For performance test, old value of given metric')
-  group.add_argument(
-      '--new_value',
-      type=float,
-      help='For performance test, new value of given metric')
-  group.add_argument(
       '--args',
       help='Extra args passed to "test_that --args"; Overrides the default')
 
@@ -306,22 +298,9 @@
 
   # Verify command line options.
   if opts.metric:
-    if opts.old_value is None:
-      logger.error('--old_value is not provided')
-      return FATAL
-    if opts.new_value is None:
-      logger.error('--new_value is not provided')
-      return FATAL
     if opts.fail_to_pass:
       logger.error('--fail_to_pass is not for benchmark test (--metric)')
       return FATAL
-  else:
-    if opts.old_value is not None:
-      logger.error('--old_value is provided but --metric is not')
-      return FATAL
-    if opts.new_value is not None:
-      logger.error('--new_value is provided but --metric is not')
-      return FATAL
   if opts.test_name.startswith('telemetry_'):
     if not opts.chrome_root:
       logger.error('--chrome_root is mandatory for telemetry tests')
@@ -345,12 +324,9 @@
       return SKIP
 
     print('BISECT_RESULT_VALUES=', ' '.join(map(str, values)))
-    average = float(sum(values)) / len(values)
-    if abs(average - opts.old_value) < abs(average - opts.new_value):
-      logger.info('values=%s, average=%s; OLD', values, average)
-      return OLD
-    logger.info('values=%s, average=%s; NEW', values, average)
-    return NEW
+    logger.info('values=%s', values)
+    # The exit code doesn't matter.
+    return OLD
   else:
     if opts.fail_to_pass:
       if passed: