Run pyformat on all the toolchain-utils files.
This gets rid of a lot of lint issues.
Ran by doing this:
for f in *.py; do echo -n "$f " ; if [ -x $f ]; then pyformat -i
--remove_trailing_comma --yapf --force_quote_type=double $f ; else
pyformat -i --remove_shebang --remove_trailing_comma --yapf
--force_quote_type=double $f ; fi ; done
BUG=chromium:567921
TEST=Ran simple crosperf run.
Change-Id: I59778835fdaa5f706d2e1765924389f9e97433d1
Reviewed-on: https://chrome-internal-review.googlesource.com/242031
Reviewed-by: Luis Lozano <llozano@chromium.org>
Commit-Queue: Luis Lozano <llozano@chromium.org>
Tested-by: Luis Lozano <llozano@chromium.org>
Reviewed-by: Yunlian Jiang <yunlian@google.com>
diff --git a/buildbot_test_toolchains.py b/buildbot_test_toolchains.py
index 8b5bce7..feb11e1 100755
--- a/buildbot_test_toolchains.py
+++ b/buildbot_test_toolchains.py
@@ -24,47 +24,52 @@
from utils import buildbot_utils
# CL that updated GCC ebuilds to use 'next_gcc'.
-USE_NEXT_GCC_PATCH = "230260"
+USE_NEXT_GCC_PATCH = '230260'
# CL that uses LLVM to build the peppy image.
-USE_LLVM_PATCH = "295217"
+USE_LLVM_PATCH = '295217'
# The boards on which we run weekly reports
-WEEKLY_REPORT_BOARDS = ["lumpy"]
+WEEKLY_REPORT_BOARDS = ['lumpy']
-CROSTC_ROOT = "/usr/local/google/crostc"
-ROLE_ACCOUNT = "mobiletc-prebuild"
+CROSTC_ROOT = '/usr/local/google/crostc'
+ROLE_ACCOUNT = 'mobiletc-prebuild'
TOOLCHAIN_DIR = os.path.dirname(os.path.realpath(__file__))
-MAIL_PROGRAM = "~/var/bin/mail-sheriff"
-WEEKLY_REPORTS_ROOT = os.path.join(CROSTC_ROOT, "weekly_test_data")
-PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, "pending_archives")
-NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, "nightly_test_reports")
+MAIL_PROGRAM = '~/var/bin/mail-sheriff'
+WEEKLY_REPORTS_ROOT = os.path.join(CROSTC_ROOT, 'weekly_test_data')
+PENDING_ARCHIVES_DIR = os.path.join(CROSTC_ROOT, 'pending_archives')
+NIGHTLY_TESTS_DIR = os.path.join(CROSTC_ROOT, 'nightly_test_reports')
+
class ToolchainComparator(object):
"""Class for doing the nightly tests work."""
- def __init__(self, board, remotes, chromeos_root, weekday,
- patches, noschedv2=False):
+ def __init__(self,
+ board,
+ remotes,
+ chromeos_root,
+ weekday,
+ patches,
+ noschedv2=False):
self._board = board
self._remotes = remotes
self._chromeos_root = chromeos_root
self._base_dir = os.getcwd()
self._ce = command_executer.GetCommandExecuter()
self._l = logger.GetLogger()
- self._build = "%s-release" % board
+ self._build = '%s-release' % board
self._patches = patches.split(',')
self._patches_string = '_'.join(str(p) for p in self._patches)
self._noschedv2 = noschedv2
if not weekday:
- self._weekday = time.strftime("%a")
+ self._weekday = time.strftime('%a')
else:
self._weekday = weekday
timestamp = datetime.datetime.strftime(datetime.datetime.now(),
- "%Y-%m-%d_%H:%M:%S")
+ '%Y-%m-%d_%H:%M:%S')
self._reports_dir = os.path.join(NIGHTLY_TESTS_DIR,
- "%s.%s" % (timestamp, board),
- )
+ '%s.%s' % (timestamp, board),)
def _ParseVanillaImage(self, trybot_image):
"""Parse a trybot artifact name to get corresponding vanilla image.
@@ -74,19 +79,19 @@
corresponding official build name, e.g. 'daisy-release/R40-6394.0.0'.
"""
start_pos = trybot_image.find(self._build)
- end_pos = trybot_image.rfind("-b")
+ end_pos = trybot_image.rfind('-b')
vanilla_image = trybot_image[start_pos:end_pos]
return vanilla_image
def _FinishSetup(self):
"""Make sure testing_rsa file is properly set up."""
# Fix protections on ssh key
- command = ("chmod 600 /var/cache/chromeos-cache/distfiles/target"
- "/chrome-src-internal/src/third_party/chromite/ssh_keys"
- "/testing_rsa")
+ command = ('chmod 600 /var/cache/chromeos-cache/distfiles/target'
+ '/chrome-src-internal/src/third_party/chromite/ssh_keys'
+ '/testing_rsa')
ret_val = self._ce.ChrootRunCommand(self._chromeos_root, command)
if ret_val != 0:
- raise RuntimeError("chmod for testing_rsa failed")
+ raise RuntimeError('chmod for testing_rsa failed')
def _TestImages(self, trybot_image, vanilla_image):
"""Create crosperf experiment file.
@@ -94,17 +99,15 @@
Given the names of the trybot and vanilla images, create the
appropriate crosperf experiment file and launch crosperf on it.
"""
- experiment_file_dir = os.path.join(self._chromeos_root, "..",
- self._weekday)
- experiment_file_name = "%s_toolchain_experiment.txt" % self._board
+ experiment_file_dir = os.path.join(self._chromeos_root, '..', self._weekday)
+ experiment_file_name = '%s_toolchain_experiment.txt' % self._board
- compiler_string = "gcc"
+ compiler_string = 'gcc'
if USE_LLVM_PATCH in self._patches_string:
- experiment_file_name = "%s_llvm_experiment.txt" % self._board
- compiler_string = "llvm"
+ experiment_file_name = '%s_llvm_experiment.txt' % self._board
+ compiler_string = 'llvm'
- experiment_file = os.path.join(experiment_file_dir,
- experiment_file_name)
+ experiment_file = os.path.join(experiment_file_dir, experiment_file_name)
experiment_header = """
board: %s
remote: %s
@@ -116,7 +119,8 @@
iterations: 3
}
"""
- with open(experiment_file, "w") as f:
+
+ with open(experiment_file, 'w') as f:
f.write(experiment_header)
f.write(experiment_tests)
@@ -130,9 +134,9 @@
""" % (self._chromeos_root, vanilla_image)
f.write(official_image)
- label_string = "%s_trybot_image" % compiler_string
+ label_string = '%s_trybot_image' % compiler_string
if USE_NEXT_GCC_PATCH in self._patches:
- label_string = "gcc_next_trybot_image"
+ label_string = 'gcc_next_trybot_image'
experiment_image = """
%s {
@@ -144,12 +148,10 @@
compiler_string)
f.write(experiment_image)
- crosperf = os.path.join(TOOLCHAIN_DIR,
- "crosperf",
- "crosperf")
+ crosperf = os.path.join(TOOLCHAIN_DIR, 'crosperf', 'crosperf')
noschedv2_opts = '--noschedv2' if self._noschedv2 else ''
- command = ("{crosperf} --no_email=True --results_dir={r_dir} "
- "--json_report=True {noschedv2_opts} {exp_file}").format(
+ command = ('{crosperf} --no_email=True --results_dir={r_dir} '
+ '--json_report=True {noschedv2_opts} {exp_file}').format(
crosperf=crosperf,
r_dir=self._reports_dir,
noschedv2_opts=noschedv2_opts,
@@ -160,7 +162,7 @@
raise RuntimeError("Couldn't run crosperf!")
else:
# Copy json report to pending archives directory.
- command = "cp %s/*.json %s/." % (self._reports_dir, PENDING_ARCHIVES_DIR)
+ command = 'cp %s/*.json %s/.' % (self._reports_dir, PENDING_ARCHIVES_DIR)
ret = self._ce.RunCommand(command)
return
@@ -175,12 +177,12 @@
dry_run = False
if os.getlogin() != ROLE_ACCOUNT:
- self._l.LogOutput("Running this from non-role account; not copying "
- "tar files for weekly reports.")
+ self._l.LogOutput('Running this from non-role account; not copying '
+ 'tar files for weekly reports.')
dry_run = True
- images_path = os.path.join(os.path.realpath(self._chromeos_root),
- "chroot/tmp")
+ images_path = os.path.join(
+ os.path.realpath(self._chromeos_root), 'chroot/tmp')
data_dir = os.path.join(WEEKLY_REPORTS_ROOT, self._board)
dest_dir = os.path.join(data_dir, self._weekday)
@@ -188,46 +190,42 @@
os.makedirs(dest_dir)
# Make sure dest_dir is empty (clean out last week's data).
- cmd = "cd %s; rm -Rf %s_*_image*" % (dest_dir, self._weekday)
+ cmd = 'cd %s; rm -Rf %s_*_image*' % (dest_dir, self._weekday)
if dry_run:
- print("CMD: %s" % cmd)
+ print('CMD: %s' % cmd)
else:
self._ce.RunCommand(cmd)
# Now create new tar files and copy them over.
- labels = ["test", "vanilla"]
+ labels = ['test', 'vanilla']
for label_name in labels:
- if label_name == "test":
+ if label_name == 'test':
test_path = trybot_image
else:
test_path = vanilla_image
- tar_file_name = "%s_%s_image.tar" % (self._weekday, label_name)
- cmd = ("cd %s; tar -cvf %s %s/chromiumos_test_image.bin; "
- "cp %s %s/.") % (images_path,
- tar_file_name,
- test_path,
- tar_file_name,
- dest_dir)
+ tar_file_name = '%s_%s_image.tar' % (self._weekday, label_name)
+ cmd = ('cd %s; tar -cvf %s %s/chromiumos_test_image.bin; '
+ 'cp %s %s/.') % (images_path, tar_file_name, test_path,
+ tar_file_name, dest_dir)
if dry_run:
- print("CMD: %s" % cmd)
+ print('CMD: %s' % cmd)
tar_ret = 0
else:
tar_ret = self._ce.RunCommand(cmd)
if tar_ret:
- self._l.LogOutput("Error while creating/copying test tar file(%s)."
- % tar_file_name)
+ self._l.LogOutput('Error while creating/copying test tar file(%s).' %
+ tar_file_name)
def _SendEmail(self):
"""Find email message generated by crosperf and send it."""
- filename = os.path.join(self._reports_dir,
- "msg_body.html")
+ filename = os.path.join(self._reports_dir, 'msg_body.html')
if (os.path.exists(filename) and
os.path.exists(os.path.expanduser(MAIL_PROGRAM))):
- email_title = "buildbot test results"
+ email_title = 'buildbot test results'
if self._patches_string == USE_LLVM_PATCH:
- email_title = "buildbot llvm test results"
- command = ('cat %s | %s -s "%s, %s" -team -html'
- % (filename, MAIL_PROGRAM, email_title, self._board))
+ email_title = 'buildbot llvm test results'
+ command = ('cat %s | %s -s "%s, %s" -team -html' %
+ (filename, MAIL_PROGRAM, email_title, self._board))
self._ce.RunCommand(command)
def DoAll(self):
@@ -237,8 +235,7 @@
crosperf, and copy images into seven-day report directories.
"""
date_str = datetime.date.today()
- description = "master_%s_%s_%s" % (self._patches_string,
- self._build,
+ description = 'master_%s_%s_%s' % (self._patches_string, self._build,
date_str)
trybot_image = buildbot_utils.GetTrybotImage(self._chromeos_root,
self._build,
@@ -248,13 +245,13 @@
vanilla_image = self._ParseVanillaImage(trybot_image)
- print ("trybot_image: %s" % trybot_image)
- print ("vanilla_image: %s" % vanilla_image)
+ print('trybot_image: %s' % trybot_image)
+ print('vanilla_image: %s' % vanilla_image)
if len(trybot_image) == 0:
- self._l.LogError("Unable to find trybot_image for %s!" % description)
+ self._l.LogError('Unable to find trybot_image for %s!' % description)
return 1
if len(vanilla_image) == 0:
- self._l.LogError("Unable to find vanilla image for %s!" % description)
+ self._l.LogError('Unable to find vanilla image for %s!' % description)
return 1
if os.getlogin() == ROLE_ACCOUNT:
self._FinishSetup()
@@ -274,51 +271,51 @@
# Common initializations
command_executer.InitCommandExecuter()
parser = optparse.OptionParser()
- parser.add_option("--remote",
- dest="remote",
- help="Remote machines to run tests on.")
- parser.add_option("--board",
- dest="board",
- default="x86-zgb",
- help="The target board.")
- parser.add_option("--chromeos_root",
- dest="chromeos_root",
- help="The chromeos root from which to run tests.")
- parser.add_option("--weekday", default="",
- dest="weekday",
- help="The day of the week for which to run tests.")
- parser.add_option("--patch",
- dest="patches",
- help="The patches to use for the testing, "
+ parser.add_option('--remote',
+ dest='remote',
+ help='Remote machines to run tests on.')
+ parser.add_option('--board',
+ dest='board',
+ default='x86-zgb',
+ help='The target board.')
+ parser.add_option('--chromeos_root',
+ dest='chromeos_root',
+ help='The chromeos root from which to run tests.')
+ parser.add_option('--weekday',
+ default='',
+ dest='weekday',
+ help='The day of the week for which to run tests.')
+ parser.add_option('--patch',
+ dest='patches',
+ help='The patches to use for the testing, '
"seprate the patch numbers with ',' "
- "for more than one patches.")
- parser.add_option("--noschedv2",
- dest="noschedv2",
- action="store_true",
+ 'for more than one patches.')
+ parser.add_option('--noschedv2',
+ dest='noschedv2',
+ action='store_true',
default=False,
- help="Pass --noschedv2 to crosperf.")
+ help='Pass --noschedv2 to crosperf.')
options, _ = parser.parse_args(argv)
if not options.board:
- print("Please give a board.")
+ print('Please give a board.')
return 1
if not options.remote:
- print("Please give at least one remote machine.")
+ print('Please give at least one remote machine.')
return 1
if not options.chromeos_root:
- print("Please specify the ChromeOS root directory.")
+ print('Please specify the ChromeOS root directory.')
return 1
if options.patches:
patches = options.patches
else:
patches = USE_NEXT_GCC_PATCH
- fc = ToolchainComparator(options.board, options.remote,
- options.chromeos_root, options.weekday, patches,
- options.noschedv2)
+ fc = ToolchainComparator(options.board, options.remote, options.chromeos_root,
+ options.weekday, patches, options.noschedv2)
return fc.DoAll()
-if __name__ == "__main__":
+if __name__ == '__main__':
retval = Main(sys.argv)
sys.exit(retval)