Restructure uploader script so imports are less likely to break.
We need to modify sys.path before we import anything from catapult.
Unfortunately we need to modify it according to --outdir, so it needs
to happen at runtime rather than import time.
I try to split the script into a main which just sets up command line
args and sys.path and then imports the main script. This makes it less
likely that future maintainers will import something too early.
Bug: chromium:1029452
Change-Id: I16bf6257269ab8ab90dd74bff7880de8b5fb8071
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/170341
Commit-Queue: Patrik Höglund <phoglund@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30788}
diff --git a/tools_webrtc/perf/catapult_uploader.py b/tools_webrtc/perf/catapult_uploader.py
new file mode 100644
index 0000000..d0b02f8
--- /dev/null
+++ b/tools_webrtc/perf/catapult_uploader.py
@@ -0,0 +1,128 @@
+#!/usr/bin/env python
+# Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS. All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+
+import httplib2
+import json
+import subprocess
+import zlib
+
+from tracing.value import histogram_set
+from tracing.value.diagnostics import generic_set
+from tracing.value.diagnostics import reserved_infos
+
+
+def _GenerateOauthToken():
+ args = ['luci-auth', 'token']
+ p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ if p.wait() == 0:
+ output = p.stdout.read()
+ return output.strip()
+ else:
+ raise RuntimeError(
+ 'Error generating authentication token.\nStdout: %s\nStderr:%s' %
+ (p.stdout.read(), p.stderr.read()))
+
+
+def _SendHistogramSet(url, histograms, oauth_token):
+ """Make a HTTP POST with the given JSON to the Performance Dashboard.
+
+ Args:
+ url: URL of Performance Dashboard instance, e.g.
+ "https://chromeperf.appspot.com".
+ histograms: a histogram set object that contains the data to be sent.
+ oauth_token: An oauth token to use for authorization.
+ """
+ headers = {'Authorization': 'Bearer %s' % oauth_token}
+
+ # TODO(https://crbug.com/1029452): HACKHACK
+ # Remove once we set bin bounds correctly in the proto writer.
+ dicts = histograms.AsDicts()
+ for d in dicts:
+ if 'name' in d:
+ d['allBins'] = [[1]]
+
+ serialized = json.dumps(dicts, indent=4)
+
+ if url.startswith('http://localhost'):
+ # The catapult server turns off compression in developer mode.
+ data = serialized
+ else:
+ data = zlib.compress(serialized)
+
+ print 'Sending %d bytes to %s.' % (len(data), url + '/add_histograms')
+
+ http = httplib2.Http()
+ response, content = http.request(url + '/add_histograms', method='POST',
+ body=data, headers=headers)
+ return response, content
+
+
+def _LoadHistogramSetFromProto(options):
+ hs = histogram_set.HistogramSet()
+ with options.input_results_file as f:
+ hs.ImportProto(f.read())
+
+ return hs
+
+
+def _AddBuildInfo(histograms, options):
+ common_diagnostics = {
+ reserved_infos.MASTERS: options.perf_dashboard_machine_group,
+ reserved_infos.BOTS: options.bot,
+ reserved_infos.POINT_ID: options.commit_position,
+ reserved_infos.BENCHMARKS: options.test_suite,
+ reserved_infos.WEBRTC_REVISIONS: str(options.webrtc_git_hash),
+ reserved_infos.BUILD_URLS: options.build_page_url,
+ }
+
+ for k, v in common_diagnostics.items():
+ histograms.AddSharedDiagnosticToAllHistograms(
+ k.name, generic_set.GenericSet([v]))
+
+
+def _DumpOutput(histograms, output_file):
+ with output_file:
+ json.dump(histograms.AsDicts(), output_file, indent=4)
+
+
+# TODO(https://crbug.com/1029452): Remove this once
+# https://chromium-review.googlesource.com/c/catapult/+/2094312 lands.
+def _HackSummaryOptions(histograms):
+ for histogram in histograms:
+ histogram.CustomizeSummaryOptions({
+ 'avg': False,
+ 'std': False,
+ 'count': False,
+ 'sum': False,
+ 'min': False,
+ 'max': False,
+ 'nans': False,
+ })
+
+
+def UploadToDashboard(options):
+ histograms = _LoadHistogramSetFromProto(options)
+ _AddBuildInfo(histograms, options)
+ _HackSummaryOptions(histograms)
+
+ if options.output_json_file:
+ _DumpOutput(histograms, options.output_json_file)
+
+ oauth_token = _GenerateOauthToken()
+ response, content = _SendHistogramSet(
+ options.dashboard_url, histograms, oauth_token)
+
+ if response.status == 200:
+ print 'Received 200 from dashboard.'
+ return 0
+ else:
+ print('Upload failed with %d: %s\n\n%s' % (response.status, response.reason,
+ content))
+ return 1