toolchain-utils: provide option to migrate nightly tests to recipe
This patch provides two new options to buildbot_test_toolchain script,
which launches nightly performance tests for toolchain.
Option `--recipe` introduces new location and mechanism to find latest
image from recipe bucket. The recipe builder for each board monitored by
toolchain team builds an image every day and store results to certain gs
locaton. When this option specified, the script no longer launches
tryjob to get a new image but tries to access the lastest existing image
in the last two days from the gs location.
Option `--test` makes it easier to test locally, it blocks all locations
in chrotomation machine and do not send or store unnecessary results
after tests finish.
A small change to lock_machine.py fixes an issue while related to
swarming when testing this change.
BUG=chromium:1071271
TEST=./buildbot_test_toolchains.py --board=lulu
--remote=chromeos2-row9-rack8-host7.cros
--chromeos_root=$chromiumos --test=True --recipe=True
Change-Id: I724a31ce174955c1f002bb7e330919f23535e105
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/third_party/toolchain-utils/+/2151717
Reviewed-by: Tiancong Wang <tcwang@google.com>
Tested-by: Zhizhou Yang <zhizhouy@google.com>
Auto-Submit: Zhizhou Yang <zhizhouy@google.com>
Commit-Queue: Zhizhou Yang <zhizhouy@google.com>
diff --git a/buildbot_test_toolchains.py b/buildbot_test_toolchains.py
index 184a1ec..1e8b33a 100755
--- a/buildbot_test_toolchains.py
+++ b/buildbot_test_toolchains.py
@@ -22,6 +22,7 @@
import datetime
import os
import re
+import shutil
import sys
import time
@@ -37,6 +38,7 @@
NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly-tests')
ROLE_ACCOUNT = 'mobiletc-prebuild'
TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
+TMP_TOOLCHAIN_TEST = '/tmp/toolchain-tests'
MAIL_PROGRAM = '~/var/bin/mail-sheriff'
PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, 'pending_archives')
NIGHTLY_TESTS_RESULTS = os.path.join(CROSTC_ROOT, 'nightly_test_reports')
@@ -45,7 +47,6 @@
IMAGE_VERSION_STR = r'{chrome_version}-{tip}\.{branch}\.{branch_branch}'
IMAGE_FS = IMAGE_DIR + '/' + IMAGE_VERSION_STR
TRYBOT_IMAGE_FS = IMAGE_FS + '-{build_id}'
-PFQ_IMAGE_FS = IMAGE_FS + '-rc1'
IMAGE_RE_GROUPS = {
'board': r'(?P<board>\S+)',
'image_type': r'(?P<image_type>\S+)',
@@ -57,6 +58,19 @@
}
TRYBOT_IMAGE_RE = TRYBOT_IMAGE_FS.format(**IMAGE_RE_GROUPS)
+RECIPE_IMAGE_FS = IMAGE_FS + '-{build_id}-{buildbucket_id}'
+RECIPE_IMAGE_RE_GROUPS = {
+ 'board': r'(?P<board>\S+)',
+ 'image_type': r'(?P<image_type>\S+)',
+ 'chrome_version': r'(?P<chrome_version>R\d+)',
+ 'tip': r'(?P<tip>\d+)',
+ 'branch': r'(?P<branch>\d+)',
+ 'branch_branch': r'(?P<branch_branch>\d+)',
+ 'build_id': r'(?P<build_id>\d+)',
+ 'buildbucket_id': r'(?P<buildbucket_id>\d+)'
+}
+RECIPE_IMAGE_RE = RECIPE_IMAGE_FS.format(**RECIPE_IMAGE_RE_GROUPS)
+
TELEMETRY_AQUARIUM_UNSUPPORTED = ['bob', 'elm', 'veyron_minnie']
@@ -69,6 +83,8 @@
chromeos_root,
weekday,
patches,
+ recipe=False,
+ test=False,
noschedv2=False):
self._board = board
self._remotes = remotes
@@ -79,6 +95,8 @@
self._build = '%s-release-tryjob' % board
self._patches = patches.split(',') if patches else []
self._patches_string = '_'.join(str(p) for p in self._patches)
+ self._recipe = recipe
+ self._test = test
self._noschedv2 = noschedv2
if not weekday:
@@ -88,7 +106,7 @@
self._date = datetime.date.today().strftime('%Y/%m/%d')
timestamp = datetime.datetime.now().strftime('%Y-%m-%d_%H:%M:%S')
self._reports_dir = os.path.join(
- NIGHTLY_TESTS_RESULTS,
+ TMP_TOOLCHAIN_TEST if self._test else NIGHTLY_TESTS_RESULTS,
'%s.%s' % (timestamp, board),
)
@@ -98,55 +116,33 @@
Args:
trybot_image: artifact name such as
'daisy-release-tryjob/R40-6394.0.0-b1389'
+ for recipe images, name is in this format:
+ 'lulu-llvm-next-nightly/R84-13037.0.0-31011-8883172717979984032/'
Returns:
Latest official image name, e.g. 'daisy-release/R57-9089.0.0'.
"""
# We need to filter out -tryjob in the trybot_image.
- trybot = re.sub('-tryjob', '', trybot_image)
- mo = re.search(TRYBOT_IMAGE_RE, trybot)
+ if self._recipe:
+ trybot = re.sub('-llvm-next-nightly', '-release', trybot_image)
+ mo = re.search(RECIPE_IMAGE_RE, trybot)
+ else:
+ trybot = re.sub('-tryjob', '', trybot_image)
+ mo = re.search(TRYBOT_IMAGE_RE, trybot)
assert mo
dirname = IMAGE_DIR.replace('\\', '').format(**mo.groupdict())
return buildbot_utils.GetLatestImage(self._chromeos_root, dirname)
- def _GetNonAFDOImageName(self, trybot_image):
- """Given a trybot artifact name, get corresponding non-AFDO image name.
-
- We get the non-AFDO image from the PFQ builders. This image
- is not generated for all the boards and, the closest PFQ image
- was the one build for the previous ChromeOS version (the chrome
- used in the current version is the one validated in the previous
- version).
- The previous ChromeOS does not always exist either. So, we try
- a couple of versions before.
-
- Args:
- trybot_image: artifact name such as
- 'daisy-release-tryjob/R40-6394.0.0-b1389'
-
- Returns:
- Corresponding chrome PFQ image name, e.g.
- 'daisy-chrome-pfq/R40-6393.0.0-rc1'.
- """
- trybot = re.sub('-tryjob', '', trybot_image)
- mo = re.search(TRYBOT_IMAGE_RE, trybot)
- assert mo
- image_dict = mo.groupdict()
- image_dict['image_type'] = 'chrome-pfq'
- for _ in range(2):
- image_dict['tip'] = str(int(image_dict['tip']) - 1)
- nonafdo_image = PFQ_IMAGE_FS.replace('\\', '').format(**image_dict)
- if buildbot_utils.DoesImageExist(self._chromeos_root, nonafdo_image):
- return nonafdo_image
- return ''
-
- def _TestImages(self, trybot_image, vanilla_image, nonafdo_image):
+ def _TestImages(self, trybot_image, vanilla_image):
"""Create crosperf experiment file.
Given the names of the trybot, vanilla and non-AFDO images, create the
appropriate crosperf experiment file and launch crosperf on it.
"""
- experiment_file_dir = os.path.join(NIGHTLY_TESTS_DIR, self._weekday)
+ if self._test:
+ experiment_file_dir = TMP_TOOLCHAIN_TEST
+ else:
+ experiment_file_dir = os.path.join(NIGHTLY_TESTS_DIR, self._weekday)
experiment_file_name = '%s_toolchain_experiment.txt' % self._board
compiler_string = 'llvm'
@@ -212,17 +208,6 @@
""" % (self._chromeos_root, vanilla_image)
f.write(official_image)
- # Now add non-AFDO image to test file.
- if nonafdo_image:
- official_nonafdo_image = """
- nonafdo_image {
- chromeos_root: %s
- build: %s
- compiler: llvm
- }
- """ % (self._chromeos_root, nonafdo_image)
- f.write(official_nonafdo_image)
-
label_string = '%s_trybot_image' % compiler_string
# Reuse autotest files from vanilla image for trybot images
@@ -240,10 +225,11 @@
crosperf = os.path.join(TOOLCHAIN_DIR, 'crosperf', 'crosperf')
noschedv2_opts = '--noschedv2' if self._noschedv2 else ''
- command = ('{crosperf} --no_email=True --results_dir={r_dir} '
+ command = ('{crosperf} --no_email={no_email} --results_dir={r_dir} '
'--intel_pstate=no_hwp --logging_level=verbose '
'--json_report=True {noschedv2_opts} {exp_file}').format(
crosperf=crosperf,
+ no_email=not self._test,
r_dir=self._reports_dir,
noschedv2_opts=noschedv2_opts,
exp_file=experiment_file)
@@ -278,32 +264,37 @@
Launch trybot, get image names, create crosperf experiment file, run
crosperf, and copy images into seven-day report directories.
"""
- buildbucket_id, trybot_image = buildbot_utils.GetTrybotImage(
- self._chromeos_root,
- self._build,
- self._patches,
- tryjob_flags=['--notests'],
- build_toolchain=True)
+ if self._recipe:
+ print('Using recipe buckets to get latest image.')
+ trybot_image = buildbot_utils.GetLatestRecipeImage(
+ self._chromeos_root, '%s-llvm-next-nightly' % self._board)
+ else:
+ # Launch tryjob and wait to get image location.
+ buildbucket_id, trybot_image = buildbot_utils.GetTrybotImage(
+ self._chromeos_root,
+ self._build,
+ self._patches,
+ tryjob_flags=['--notests'],
+ build_toolchain=True)
+ print('trybot_url: \
+ http://cros-goldeneye/chromeos/healthmonitoring/buildDetails?buildbucketId=%s'
+ % buildbucket_id)
- print('trybot_url: \
- http://cros-goldeneye/chromeos/healthmonitoring/buildDetails?buildbucketId=%s'
- % buildbucket_id)
if not trybot_image:
self._l.LogError('Unable to find trybot_image!')
return 2
vanilla_image = self._GetVanillaImageName(trybot_image)
- nonafdo_image = self._GetNonAFDOImageName(trybot_image)
print('trybot_image: %s' % trybot_image)
print('vanilla_image: %s' % vanilla_image)
- print('nonafdo_image: %s' % nonafdo_image)
- ret = self._TestImages(trybot_image, vanilla_image, nonafdo_image)
+ ret = self._TestImages(trybot_image, vanilla_image)
# Always try to send report email as crosperf will generate report when
# tests partially succeeded.
- self._SendEmail()
- self._CopyJson()
+ if not self._test:
+ self._SendEmail()
+ self._CopyJson()
# Non-zero ret here means crosperf tests partially failed, raise error here
# so that toolchain summary report can catch it.
if ret != 0:
@@ -343,6 +334,19 @@
action='store_true',
default=False,
help='Pass --noschedv2 to crosperf.')
+ parser.add_argument(
+ '--recipe',
+ dest='recipe',
+ default=True,
+ help='Use images generated from recipe rather than'
+ 'launching tryjob to get images.')
+ parser.add_argument(
+ '--test',
+ dest='test',
+ default=False,
+ help='Test this script on local desktop, '
+ 'disabling mobiletc checking and email sending.'
+ 'Artifacts stored in /tmp/toolchain-tests')
options = parser.parse_args(argv[1:])
if not options.board:
@@ -354,9 +358,15 @@
if not options.chromeos_root:
print('Please specify the ChromeOS root directory.')
return 1
+ if options.test:
+ print('Cleaning local test directory for this script.')
+ if os.path.exists(TMP_TOOLCHAIN_TEST):
+ shutil.rmtree(TMP_TOOLCHAIN_TEST)
+ os.mkdir(TMP_TOOLCHAIN_TEST)
fc = ToolchainComparator(options.board, options.remote, options.chromeos_root,
- options.weekday, options.patches, options.noschedv2)
+ options.weekday, options.patches, options.recipe,
+ options.test, options.noschedv2)
return fc.DoAll()