blob: 3f7136ccfe04809fba463d261f9cec403aff3fcd [file] [log] [blame]
Kuang-che Wu32f27242019-05-16 17:34:50 +08001#!/usr/bin/env python2
2# -*- coding: utf-8 -*-
3# Copyright 2019 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6"""Evaluate ChromeOS tast tests."""
7from __future__ import print_function
8import argparse
9import json
10import logging
11import os
12import re
Kuang-che Wu6c51d302019-06-19 09:07:20 +080013import shutil
Kuang-che Wu32f27242019-05-16 17:34:50 +080014import subprocess
15import sys
16
17from bisect_kit import catapult_util
18from bisect_kit import cli
19from bisect_kit import common
20from bisect_kit import configure
21from bisect_kit import cros_util
Kuang-che Wu11713052019-05-30 16:21:54 +080022from bisect_kit import util
Kuang-che Wu32f27242019-05-16 17:34:50 +080023
24logger = logging.getLogger(__name__)
25
26OLD = 'old'
27NEW = 'new'
28SKIP = 'skip'
29FATAL = 'fatal'
30
31EXIT_CODE_MAP = {
32 OLD: cli.EXIT_CODE_OLD,
33 NEW: cli.EXIT_CODE_NEW,
34 SKIP: cli.EXIT_CODE_SKIP,
35 FATAL: cli.EXIT_CODE_FATAL,
36}
37
38
39def create_argument_parser():
40 parser = argparse.ArgumentParser(description=__doc__)
Kuang-che Wufe1e88a2019-09-10 21:52:25 +080041 cli.patching_argparser_exit(parser)
Kuang-che Wu32f27242019-05-16 17:34:50 +080042 common.add_common_arguments(parser)
43 parser.add_argument(
44 'dut',
45 nargs='?',
46 type=cli.argtype_notempty,
47 metavar='DUT',
48 default=configure.get('DUT', ''))
49 parser.add_argument(
50 '--chromeos_root',
51 type=cli.argtype_dir_path,
52 metavar='CHROMEOS_ROOT',
53 default=configure.get('CHROMEOS_ROOT', ''),
54 help='ChromeOS tree root')
55 parser.add_argument(
56 '--tast_build',
57 action='store_true',
58 help='Build tast test bundle (-build=true) if specified; '
59 'default is using prebuilt bundle on the DUT')
60 parser.add_argument(
Kuang-che Wu4fc0f1c2019-08-02 21:56:49 +080061 '--with_private_bundles',
62 action='store_true',
63 help='Whether search tests in private bundles or not')
64 parser.add_argument(
Kuang-che Wu32f27242019-05-16 17:34:50 +080065 '--reboot_before_test',
66 action='store_true',
67 help='Reboot before test run')
68
69 group = parser.add_argument_group(title='Options for normal autotest tests')
70 group.add_argument(
71 '--test_name',
72 required=True,
73 help='Test name, like "video_VideoDecodeAccelerator.h264"')
74 group.add_argument(
75 '--fail_to_pass',
76 action='store_true',
77 help='For functional tests: old behavior is FAIL and new behavior is '
78 'PASS; If not specified, default = old behavior is PASS and new '
79 'behavior is FAIL')
80 group.add_argument(
81 '--metric',
82 help=
83 'Metric name of performance test; example: "cheets_SystemRawImageSize"')
84
85 return parser
86
87
88def prepare_to_run_test(opts):
89 if opts.reboot_before_test:
90 cros_util.reboot(opts.dut)
91
92
Kuang-che Wu4fc0f1c2019-08-02 21:56:49 +080093def get_tast_bundle_info(opts, pattern=None):
94 bundles = ['cros']
95 flags = ['-json']
Kuang-che Wu11713052019-05-30 16:21:54 +080096 if not opts.tast_build:
Kuang-che Wu4fc0f1c2019-08-02 21:56:49 +080097 flags.append('-build=false')
98 if opts.with_private_bundles:
99 flags.append('-downloadprivatebundles=true')
100 else:
101 if opts.with_private_bundles:
102 bundles.append('crosint')
103
104 args = [opts.dut]
Kuang-che Wu11713052019-05-30 16:21:54 +0800105 if pattern:
Kuang-che Wu4fc0f1c2019-08-02 21:56:49 +0800106 args.append(pattern)
107
108 result = {}
109 for bundle in bundles:
110 cmd = ['tast', 'list'] + flags + ['-buildbundle=' + bundle] + args
Kuang-che Wubcafc552019-08-15 15:27:02 +0800111 json_text = cros_util.cros_sdk(opts.chromeos_root, *cmd, log_stdout=False)
Kuang-che Wu4fc0f1c2019-08-02 21:56:49 +0800112 for entry in json.loads(json_text):
113 result[entry['name']] = bundle
114
115 return result
Kuang-che Wu11713052019-05-30 16:21:54 +0800116
117
Kuang-che Wu4fc0f1c2019-08-02 21:56:49 +0800118def run_test(opts, bundle):
Kuang-che Wu32f27242019-05-16 17:34:50 +0800119 """Runs an autotest test.
120
121 Args:
122 opts: An argparse.Namespace to hold command line arguments.
Kuang-che Wu4fc0f1c2019-08-02 21:56:49 +0800123 bundle: tast's test bundle
Kuang-che Wu32f27242019-05-16 17:34:50 +0800124
125 Returns:
126 path of test result (outside chroot)
127 """
128 # Set results dir inside source tree, so it's easier to access them outside
129 # chroot.
130 results_dir = os.path.join(cros_util.chromeos_root_inside_chroot,
131 'tmp/tast_results_tmp')
Kuang-che Wu6c51d302019-06-19 09:07:20 +0800132 results_dir_output_chroot = results_dir.replace(
133 cros_util.chromeos_root_inside_chroot, opts.chromeos_root)
134 # Don't reuse existing results dir, otherwise tast may rename output files.
135 if os.path.exists(results_dir_output_chroot):
136 shutil.rmtree(results_dir_output_chroot)
137
Kuang-che Wu32f27242019-05-16 17:34:50 +0800138 # TODO(kcwu): add -timeout
139 cmd = ['tast', '-verbose', 'run']
140 if not opts.tast_build:
141 cmd.append('-build=false')
Kuang-che Wu4fc0f1c2019-08-02 21:56:49 +0800142 if opts.with_private_bundles:
143 cmd.append('-downloadprivatebundles=true')
144 else:
145 cmd.append('-buildbundle=' + bundle)
Kuang-che Wu32f27242019-05-16 17:34:50 +0800146 cmd += ['-resultsdir', results_dir, opts.dut, opts.test_name]
147
Kuang-che Wu34a67542019-09-09 18:01:05 +0800148 cros_util.cros_sdk(opts.chromeos_root, *cmd)
Kuang-che Wu32f27242019-05-16 17:34:50 +0800149
Kuang-che Wu6c51d302019-06-19 09:07:20 +0800150 return results_dir_output_chroot
Kuang-che Wu32f27242019-05-16 17:34:50 +0800151
152
153def gather_test_result(opts, result_dir):
154 error_path = os.path.join(result_dir, 'run_error.txt')
155 if os.path.exists(error_path):
156 message = open(error_path).read()
157 raise Exception('tast global error: %s' % message)
158
159 results_path = os.path.join(result_dir, 'results.json')
160 passed = None
161 for result in json.load(open(results_path)):
162 if result['name'] != opts.test_name:
163 logger.warning('unexpected test ran: %s', result['name'])
164 continue
165 passed = result['errors'] is None
166 if passed is None:
167 raise Exception('no test result for "%s"?' % opts.test_name)
168
169 values = []
170 if opts.metric:
171 chart_path = os.path.join(result_dir, 'tests', opts.test_name,
172 'results-chart.json')
173 values = catapult_util.get_benchmark_values(chart_path, opts.metric)
174
175 return passed, values
176
177
Kuang-che Wufe1e88a2019-09-10 21:52:25 +0800178@cli.fatal_error_handler
Kuang-che Wu32f27242019-05-16 17:34:50 +0800179def main(args=None):
180 common.init()
181 parser = create_argument_parser()
182 opts = parser.parse_args(args)
183 common.config_logging(opts)
184
185 if not cros_util.is_dut(opts.dut):
186 logger.error('%r is not a valid DUT address', opts.dut)
187 return FATAL
188
Kuang-che Wu4fc0f1c2019-08-02 21:56:49 +0800189 if opts.with_private_bundles and not opts.tast_build:
190 version = cros_util.query_dut_short_version(opts.dut)
191 if cros_util.is_cros_localbuild_version(version):
192 logger.error(
193 'for non-official chromeos image, --tast_build must be specified')
194 return FATAL
195
Kuang-che Wu32f27242019-05-16 17:34:50 +0800196 # Verify command line options.
197 if opts.metric:
198 if opts.fail_to_pass:
199 logger.error('--fail_to_pass is not for benchmark test (--metric)')
200 return FATAL
201 # Remove the "tast." prefix prepended by autotest.
202 opts.test_name = re.sub(r'^tast\.', '', opts.test_name)
203
Kuang-che Wu4fc0f1c2019-08-02 21:56:49 +0800204 tast_bundle_info = get_tast_bundle_info(opts, opts.test_name)
205 if not tast_bundle_info:
206 tast_bundle_info = get_tast_bundle_info(opts)
207 util.show_similar_candidates('test name', opts.test_name,
208 list(tast_bundle_info))
209 return FATAL
210
211 if len(tast_bundle_info) != 1 or opts.test_name not in tast_bundle_info:
212 # For example, tast in chroot after 12205.0.0 is IPC incompatible with
213 # tast on DUT earlier than 12028.0.0 (crbug/932307)
214 logger.fatal('"tast list" returns unexpected tests; '
215 'incompatible tast on DUT and in chroot?')
Kuang-che Wu11713052019-05-30 16:21:54 +0800216 return FATAL
217
Kuang-che Wu32f27242019-05-16 17:34:50 +0800218 try:
219 prepare_to_run_test(opts)
220 except Exception:
221 logger.exception('failed when prepare, assume it is temporary; SKIP')
222 return SKIP
223
Kuang-che Wu4fc0f1c2019-08-02 21:56:49 +0800224 bundle = tast_bundle_info[opts.test_name]
Kuang-che Wu34a67542019-09-09 18:01:05 +0800225 try:
226 result_dir = run_test(opts, bundle)
227 except subprocess.CalledProcessError:
228 logger.error('failed to run tast; maybe build, ssh, or setup failures')
229 return SKIP
Kuang-che Wu32f27242019-05-16 17:34:50 +0800230
231 try:
232 passed, values = gather_test_result(opts, result_dir)
233 except Exception:
234 logger.exception('failed to parse test result')
235 return FATAL
236
237 if opts.metric:
238 if not values:
239 logger.warning('no values found; SKIP')
240 return SKIP
241
242 print('BISECT_RESULT_VALUES=', ' '.join(map(str, values)))
243 logger.info('values=%s', values)
244 # The exit code doesn't matter.
245 return OLD
246 else:
247 if opts.fail_to_pass:
248 if passed:
249 logger.info('passed')
250 return NEW
251 logger.info('failed')
252 return OLD
253 else:
254 if passed:
255 logger.info('passed')
256 return OLD
257 logger.info('failed')
258 return NEW
259
260
261if __name__ == '__main__':
262 sys.exit(EXIT_CODE_MAP[main()])