blob: d21ad5a4c9a1ff2996a8228f42e37ad24df5ae5b [file] [log] [blame]
xixuan878b1eb2017-03-20 15:58:17 -07001# Copyright 2017 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Module for interacting with google APIs."""
Xixuan Wu5d6063e2017-09-05 16:15:07 -07006# pylint: disable=g-bad-import-order
Xixuan Wud55ac6e2019-03-14 10:56:39 -07007# pylint: disable=g-bad-exception-name
Xixuan Wu5d6063e2017-09-05 16:15:07 -07008
Xixuan Wuf856ff12019-05-21 14:09:38 -07009import ast
Xixuan Wu5d6063e2017-09-05 16:15:07 -070010import httplib2
Xixuan Wu55d38c52019-05-21 14:26:23 -070011import logging
Xixuan Wuf856ff12019-05-21 14:09:38 -070012import re
xixuan878b1eb2017-03-20 15:58:17 -070013
14import apiclient
Xixuan Wuf856ff12019-05-21 14:09:38 -070015import build_lib
xixuan878b1eb2017-03-20 15:58:17 -070016import constants
17import file_getter
Xixuan Wuf856ff12019-05-21 14:09:38 -070018import global_config
19import time_converter
xixuan878b1eb2017-03-20 15:58:17 -070020
21from oauth2client import service_account
22from oauth2client.contrib import appengine
23
24
25class RestClientError(Exception):
Xixuan Wu5d6063e2017-09-05 16:15:07 -070026 """Raised when there is a general error."""
xixuan878b1eb2017-03-20 15:58:17 -070027
28
29class NoServiceRestClientError(RestClientError):
Xixuan Wu5d6063e2017-09-05 16:15:07 -070030 """Raised when there is no ready service for a google API."""
xixuan878b1eb2017-03-20 15:58:17 -070031
32
33class BaseRestClient(object):
Xixuan Wu5d6063e2017-09-05 16:15:07 -070034 """Base class of REST client for google APIs."""
xixuan878b1eb2017-03-20 15:58:17 -070035
Xixuan Wu5d6063e2017-09-05 16:15:07 -070036 def __init__(self, scopes, service_name, service_version):
37 """Initialize a REST client to connect to a google API.
xixuan878b1eb2017-03-20 15:58:17 -070038
Xixuan Wu5d6063e2017-09-05 16:15:07 -070039 Args:
40 scopes: the scopes of the to-be-connected API.
41 service_name: the service name of the to-be-connected API.
42 service_version: the service version of the to-be-connected API.
43 """
44 self.running_env = constants.environment()
45 self.scopes = scopes
46 self.service_name = service_name
47 self.service_version = service_version
xixuan878b1eb2017-03-20 15:58:17 -070048
Xixuan Wu5d6063e2017-09-05 16:15:07 -070049 @property
50 def service(self):
51 if not self._service:
52 raise NoServiceRestClientError('No service created for calling API')
xixuan878b1eb2017-03-20 15:58:17 -070053
Xixuan Wu5d6063e2017-09-05 16:15:07 -070054 return self._service
xixuan878b1eb2017-03-20 15:58:17 -070055
Xixuan Wu5d6063e2017-09-05 16:15:07 -070056 def create_service(self, discovery_url=None):
57 """Create the service for a google API."""
58 self._init_credentials()
59 # Explicitly specify timeout for http to avoid DeadlineExceededError.
60 # It's used for services like AndroidBuild API, which raise such error
61 # when being triggered too many calls in a short time frame.
62 # http://stackoverflow.com/questions/14698119/httpexception-deadline-exceeded-while-waiting-for-http-response-from-url-dead
63 http_auth = self._credentials.authorize(httplib2.Http(timeout=30))
64 if discovery_url is None:
65 self._service = apiclient.discovery.build(
66 self.service_name, self.service_version,
67 http=http_auth)
68 else:
69 self._service = apiclient.discovery.build(
70 self.service_name, self.service_version, http=http_auth,
71 discoveryServiceUrl=discovery_url)
xixuan878b1eb2017-03-20 15:58:17 -070072
Xixuan Wu5d6063e2017-09-05 16:15:07 -070073 def _init_credentials(self):
74 """Initialize the credentials for a google API."""
75 if (self.running_env == constants.RunningEnv.ENV_STANDALONE or
76 self.running_env == constants.RunningEnv.ENV_DEVELOPMENT_SERVER):
77 # Running locally
78 service_credentials = service_account.ServiceAccountCredentials
79 self._credentials = service_credentials.from_json_keyfile_name(
Xixuan Wu26d06e02017-09-20 14:50:28 -070080 file_getter.STAGING_CLIENT_SECRETS_FILE, self.scopes)
Xixuan Wu5d6063e2017-09-05 16:15:07 -070081 else:
82 # Running in app-engine production
83 self._credentials = appengine.AppAssertionCredentials(self.scopes)
xixuan878b1eb2017-03-20 15:58:17 -070084
85
86class AndroidBuildRestClient(object):
Xixuan Wu5d6063e2017-09-05 16:15:07 -070087 """REST client for android build API."""
xixuan878b1eb2017-03-20 15:58:17 -070088
Xixuan Wu5d6063e2017-09-05 16:15:07 -070089 def __init__(self, rest_client):
90 """Initialize a REST client for connecting to Android Build API."""
91 self._rest_client = rest_client
92 self._rest_client.create_service()
xixuan878b1eb2017-03-20 15:58:17 -070093
Xixuan Wu5d6063e2017-09-05 16:15:07 -070094 def get_latest_build_id(self, branch, target):
95 """Get the latest build id for a given branch and target.
xixuan878b1eb2017-03-20 15:58:17 -070096
Xixuan Wu5d6063e2017-09-05 16:15:07 -070097 Args:
98 branch: an android build's branch
99 target: an android build's target
xixuan878b1eb2017-03-20 15:58:17 -0700100
Xixuan Wu5d6063e2017-09-05 16:15:07 -0700101 Returns:
102 A string representing latest build id.
103 """
104 request = self._rest_client.service.build().list(
105 buildType='submitted',
106 branch=branch,
107 target=target,
108 successful=True,
109 maxResults=1)
110 builds = request.execute(num_retries=10)
111 if not builds or not builds['builds']:
112 return None
xixuan878b1eb2017-03-20 15:58:17 -0700113
Xixuan Wu5d6063e2017-09-05 16:15:07 -0700114 return builds['builds'][0]['buildId']
xixuan878b1eb2017-03-20 15:58:17 -0700115
116
117class StorageRestClient(object):
Xixuan Wu5d6063e2017-09-05 16:15:07 -0700118 """REST client for google storage API."""
xixuan878b1eb2017-03-20 15:58:17 -0700119
Xixuan Wu5d6063e2017-09-05 16:15:07 -0700120 def __init__(self, rest_client):
121 """Initialize a REST client for connecting to Google storage API."""
122 self._rest_client = rest_client
123 self._rest_client.create_service()
xixuan878b1eb2017-03-20 15:58:17 -0700124
Xixuan Wud55ac6e2019-03-14 10:56:39 -0700125 def read_object(self, bucket, object_path):
Xixuan Wu5d6063e2017-09-05 16:15:07 -0700126 """Read the contents of input_object in input_bucket.
xixuan878b1eb2017-03-20 15:58:17 -0700127
Xixuan Wu5d6063e2017-09-05 16:15:07 -0700128 Args:
Xixuan Wud55ac6e2019-03-14 10:56:39 -0700129 bucket: A string to indicate the bucket for fetching the object.
130 e.g. constants.StorageBucket.PROD_SUITE_SCHEDULER
131 object_path: A string to indicate the path of the object to read the
132 contents.
xixuan878b1eb2017-03-20 15:58:17 -0700133
Xixuan Wu5d6063e2017-09-05 16:15:07 -0700134 Returns:
135 the stripped string contents of the input object.
xixuan878b1eb2017-03-20 15:58:17 -0700136
Xixuan Wu5d6063e2017-09-05 16:15:07 -0700137 Raises:
138 apiclient.errors.HttpError
139 """
140 req = self._rest_client.service.objects().get_media(
Xixuan Wud55ac6e2019-03-14 10:56:39 -0700141 bucket=bucket,
142 object=object_path)
143 return req.execute()
144
145 def upload_object(self, bucket, src_object_path, dest_object_path):
146 """Upload object_path to input_bucket.
147
148 Args:
149 bucket: A string to indicate the bucket for the object to be uploaded to.
150 src_object_path: A string the full path of the object to upload.
151 dest_object_path: A string path inside bucket to upload to.
152
153 Returns:
154 A dict of uploaded object info.
155
156 Raises:
157 apiclient.errors.HttpError
158 """
159 req = self._rest_client.service.objects().insert(
160 bucket=bucket,
161 name=dest_object_path,
162 media_body=src_object_path,
163 media_mime_type='text/plain',
164 )
Xixuan Wu5d6063e2017-09-05 16:15:07 -0700165 return req.execute()
xixuan878b1eb2017-03-20 15:58:17 -0700166
167
168class CalendarRestClient(object):
Xixuan Wu5d6063e2017-09-05 16:15:07 -0700169 """Class of REST client for google calendar API."""
xixuan878b1eb2017-03-20 15:58:17 -0700170
Xixuan Wu5d6063e2017-09-05 16:15:07 -0700171 def __init__(self, rest_client):
172 """Initialize a REST client for connecting to Google calendar API."""
173 self._rest_client = rest_client
174 self._rest_client.create_service()
xixuan878b1eb2017-03-20 15:58:17 -0700175
Xixuan Wu5d6063e2017-09-05 16:15:07 -0700176 def add_event(self, calendar_id, input_event):
177 """Add events of a given calendar.
xixuan878b1eb2017-03-20 15:58:17 -0700178
Xixuan Wu5d6063e2017-09-05 16:15:07 -0700179 Args:
180 calendar_id: the ID of the given calendar.
181 input_event: the event to be added.
182 """
183 self._rest_client.service.events().insert(
184 calendarId=calendar_id,
185 body=input_event).execute()
xixuan878b1eb2017-03-20 15:58:17 -0700186
187
Xixuan Wu6f117e92017-10-27 10:51:58 -0700188class StackdriverRestClient(object):
189 """REST client for google storage API."""
190
191 def __init__(self, rest_client):
192 """Initialize a REST client for connecting to Google storage API."""
193 self._rest_client = rest_client
194 self._rest_client.create_service()
195
196 def read_logs(self, request):
Xinan Lin318cf752019-07-19 14:50:23 -0700197 # project_id, page_size, order_by, query_filter=''):
Xixuan Wu6f117e92017-10-27 10:51:58 -0700198 """Read the logs of the project_id based on all filters.
199
200 Args:
201 request: a request dict generated by
202 stackdriver_lib.form_logging_client_request.
203
204 Returns:
205 A json object, can be parsed by
206 stackdriver_lib.parse_logging_client_response.
207
208 Raises:
209 apiclient.errors.HttpError
210 """
211 req = self._rest_client.service.entries().list(
212 fields='entries/protoPayload', body=request)
213 return req.execute()
214
215
xixuan878b1eb2017-03-20 15:58:17 -0700216class SwarmingRestClient(object):
Xixuan Wu5d6063e2017-09-05 16:15:07 -0700217 """REST client for swarming proxy API."""
xixuan878b1eb2017-03-20 15:58:17 -0700218
Xixuan Wu5d6063e2017-09-05 16:15:07 -0700219 DISCOVERY_URL_PATTERN = '%s/discovery/v1/apis/%s/%s/rest'
xixuan878b1eb2017-03-20 15:58:17 -0700220
Xixuan Wu5d6063e2017-09-05 16:15:07 -0700221 def __init__(self, rest_client, service_url):
222 self._rest_client = rest_client
223 discovery_url = self.DISCOVERY_URL_PATTERN % (
224 service_url, rest_client.service_name, rest_client.service_version)
225 self._rest_client.create_service(discovery_url=discovery_url)
xixuan878b1eb2017-03-20 15:58:17 -0700226
Xixuan Wu5d6063e2017-09-05 16:15:07 -0700227 def create_task(self, request):
228 """Create new task.
xixuan878b1eb2017-03-20 15:58:17 -0700229
Xixuan Wu5d6063e2017-09-05 16:15:07 -0700230 Args:
231 request: a json-compatible dict expected by swarming server.
232 See _to_raw_request's output in swarming_lib.py for details.
xixuan878b1eb2017-03-20 15:58:17 -0700233
Xixuan Wu5d6063e2017-09-05 16:15:07 -0700234 Returns:
235 A json dict returned by API task.new.
236 """
237 return self._rest_client.service.tasks().new(
238 fields='request,task_id', body=request).execute()
xixuan878b1eb2017-03-20 15:58:17 -0700239
Xixuan Wu5d6063e2017-09-05 16:15:07 -0700240 def get_task_result(self, task_id):
241 """Get task results by a given task_id.
xixuan878b1eb2017-03-20 15:58:17 -0700242
Xixuan Wu5d6063e2017-09-05 16:15:07 -0700243 Args:
244 task_id: A string, represents task id.
245
246 Returns:
247 A json dict returned by API task.result.
248 """
249 return self._rest_client.service.task().result(
250 task_id=task_id).execute()
Xixuan Wu7d142a92019-04-26 12:03:02 -0700251
252
253class BigqueryRestClient(object):
254 """Class of REST client for Bigquery API."""
255
Xixuan Wu55d38c52019-05-21 14:26:23 -0700256 PROJECT_TO_RUN_BIGQUERY_JOB = 'google.com:suite-scheduler'
257
Xinan Linc9f01152020-02-05 22:05:13 -0800258 def __init__(self, rest_client, project=None, dataset=None, table=None):
Xixuan Wu7d142a92019-04-26 12:03:02 -0700259 """Initialize a REST client for connecting to Bigquery API."""
260 self._rest_client = rest_client
261 self._rest_client.create_service()
Xinan Linc9f01152020-02-05 22:05:13 -0800262 self.project = project
263 self.dataset = dataset
264 self.table = table
Xixuan Wu7d142a92019-04-26 12:03:02 -0700265
Xixuan Wu55d38c52019-05-21 14:26:23 -0700266 def query(self, query_str):
267 """Query bigquery.
Xixuan Wu7d142a92019-04-26 12:03:02 -0700268
269 Args:
Xixuan Wu55d38c52019-05-21 14:26:23 -0700270 query_str: A string used to query Bigquery.
Xixuan Wu7d142a92019-04-26 12:03:02 -0700271
272 Returns:
273 A json dict returned by API bigquery.jobs.query, e.g.
274 # {...,
275 # "rows": [
276 # {
277 # "f": [ # field
278 # {
279 # "v": # value
Xixuan Wu55d38c52019-05-21 14:26:23 -0700280 # },
281 # {
282 # "v": # value
283 # },
284 # ...
Xixuan Wu7d142a92019-04-26 12:03:02 -0700285 # ]
286 # }
Xixuan Wuf856ff12019-05-21 14:09:38 -0700287 # ...
Xixuan Wu7d142a92019-04-26 12:03:02 -0700288 # ]
289 # }
290 """
291 query_data = {
Xixuan Wu55d38c52019-05-21 14:26:23 -0700292 'query': query_str,
293 'useLegacySql': False,
Xixuan Wu7d142a92019-04-26 12:03:02 -0700294 }
295 return self._rest_client.service.jobs().query(
Xixuan Wu55d38c52019-05-21 14:26:23 -0700296 projectId=self.PROJECT_TO_RUN_BIGQUERY_JOB,
297 fields='rows',
Xixuan Wu7d142a92019-04-26 12:03:02 -0700298 body=query_data).execute()
Xixuan Wu55d38c52019-05-21 14:26:23 -0700299
Xinan Linc9f01152020-02-05 22:05:13 -0800300 def insert(self, rows):
301 """Insert rows to specified Bigquery table.
302
303 Args:
304 rows: list of json objects.
305
306 Raise:
307 RestClientError: if project/dataset/table is not defined.
308 """
309 if not any([self.project, self.dataset, self.table]):
310 raise RestClientError('Project, dataset, table should be all set.'
311 'Got project:%s, dataset:%s, table:%s' %
312 (self.project, self.dataset, self.table))
313 body = {
314 'kind': 'bigquery#tableDataInsertAllRequest',
315 'rows': rows,
316 }
317 request = self._rest_client.service.tabledata().insertAll(
318 projectId=self.project,
319 datasetId=self.dataset,
320 tableId=self.table,
321 body=body)
322 response = request.execute(num_retries=3)
323 if response.get('insertErrors'):
324 logging.error('InsertRequest reported errors: %r',
325 response.get('insertErrors'))
326 return False
327
328 return True
Xixuan Wu55d38c52019-05-21 14:26:23 -0700329
Xinan Lin80a9d932019-10-17 09:24:43 -0700330class CrOSTestPlatformBigqueryClient(BigqueryRestClient):
331 """REST client for cros_test_platform builder Bigquery API."""
Xixuan Wu55d38c52019-05-21 14:26:23 -0700332
Xinan Lin80a9d932019-10-17 09:24:43 -0700333 def get_past_job_nums(self, hours):
334 """Query the count of the jobs kicked off to cros_test_platform.
Xixuan Wu55d38c52019-05-21 14:26:23 -0700335
336 Args:
337 hours: An integer.
338
339 Returns:
340 An integer.
341 """
342 query_str = """
343 SELECT
344 COUNT(*)
345 FROM
Xinan Lin80a9d932019-10-17 09:24:43 -0700346 `cr-buildbucket.chromeos.builds`
Xixuan Wu55d38c52019-05-21 14:26:23 -0700347 WHERE
Xinan Lin80a9d932019-10-17 09:24:43 -0700348 created_by = 'user:suite-scheduler.google.com@appspot.gserviceaccount.com'
349 and create_time >= TIMESTAMP_SUB(CURRENT_TIMESTAMP(), INTERVAL %d HOUR);
Xixuan Wu55d38c52019-05-21 14:26:23 -0700350 """
351 res = self.query(query_str % hours)
352 try:
353 return int(_parse_bq_job_query(res)[0][0])
354 except (ValueError, KeyError) as e:
355 logging.debug('The returned json: \n%r', res)
356 logging.exception(str(e))
357 raise
358
359
Xixuan Wuf856ff12019-05-21 14:09:38 -0700360class BuildBucketBigqueryClient(BigqueryRestClient):
361 """Rest client for buildbucket Bigquery API."""
362
Xinan Lin028f9582019-12-11 10:55:33 -0800363 def get_latest_passed_firmware_builds(self):
364 """Get artifact link of the latest passed firmware builds for board.
Xixuan Wuf856ff12019-05-21 14:09:38 -0700365
Xinan Lin028f9582019-12-11 10:55:33 -0800366 The query returns the latest firmware build for the combination of
367 board and build spec, which is cros or firmware. No restriction set
368 in the query, so it should return all available builds.
Xixuan Wuf856ff12019-05-21 14:09:38 -0700369
370 Returns:
Xinan Lin028f9582019-12-11 10:55:33 -0800371 A list of (spec, board, firmware_artifact_link).
Xixuan Wuf856ff12019-05-21 14:09:38 -0700372 """
373 query_str = """
374 SELECT
Xinan Lin028f9582019-12-11 10:55:33 -0800375 spec,
376 board,
377 /*
378 * Firmware builds may contain artifacts for multiple boards in a
379 * single build - each in a separate directory.
380 */
381 IF(spec = 'firmware', CONCAT(artifact, '/', board), artifact) as artifact
Xixuan Wuf856ff12019-05-21 14:09:38 -0700382 FROM
Xinan Lin028f9582019-12-11 10:55:33 -0800383 (
384 SELECT
385 spec,
386 board,
387 artifact,
388 RANK() OVER (PARTITION BY spec, board ORDER BY end_time DESC) AS rank
389 FROM
390 (
391 SELECT
392 /*
393 * build_config is a string contains the board and build type.
394 * For Cros build, it has the form of "BoardName-release", while
395 * the firmware config shows like "firmware-BoardName-[1]-firmwarebranch".
396 * [1] is the firmware ver.
397 */
398 IF(prefix = 'firmware', 'firmware', 'cros') AS spec,
399 IF(prefix = 'firmware', firmware_post_prefix, cros_prefix) AS board,
400 artifact,
401 end_time
402 FROM
403 (
404 SELECT
405 SPLIT(build_config, '-') [OFFSET(0)] AS prefix,
406 SPLIT(build_config, '-') [OFFSET(1)] AS firmware_post_prefix,
407 REGEXP_EXTRACT(
408 build_config, r"(^[a-zA-Z0-9_.+-]+)-release"
409 ) as cros_prefix,
410 end_time,
411 artifact
412 FROM
413 (
414 SELECT
Sean McAllister53dd3d82021-05-18 15:15:14 -0600415 COALESCE(
416 JSON_EXTRACT_SCALAR(
417 output.properties, '$.artifact_link'
418 ),
419 FORMAT('gs://%s/%s',
420 JSON_EXTRACT_SCALAR(output.properties, '$.artifacts.gs_bucket'),
421 JSON_EXTRACT_SCALAR(output.properties, '$.artifacts.gs_path'))
Sean McAllister3fb4cec2021-04-20 22:38:38 +0000422 ) as artifact,
Sean McAllister53dd3d82021-05-18 15:15:14 -0600423 COALESCE(
424 JSON_EXTRACT_SCALAR(
425 output.properties, '$.cbb_config'
426 ),
427 builder.builder
Xinan Lin028f9582019-12-11 10:55:33 -0800428 ) as build_config,
429 end_time
430 FROM `cr-buildbucket.chromeos.completed_builds_BETA`
431 WHERE
432 status = 'SUCCESS'
433 AND JSON_EXTRACT_SCALAR(
434 output.properties, '$.suite_scheduling'
435 ) = 'True'
436 )
437 )
438 )
439 )
440 WHERE rank = 1
Xixuan Wuf856ff12019-05-21 14:09:38 -0700441 """
Xinan Lin028f9582019-12-11 10:55:33 -0800442 res = self.query(query_str)
Xixuan Wuf856ff12019-05-21 14:09:38 -0700443 res = _parse_bq_job_query(res)
444 if res is None:
445 return None
Xinan Lin028f9582019-12-11 10:55:33 -0800446 logging.info('Fetched the latest artifact links: %s',
447 [row[2] for row in res])
448 return res
Xinan Lin318cf752019-07-19 14:50:23 -0700449
Xinan Lin71eeeb02020-03-10 17:37:12 -0700450 def get_passed_builds(self, earliest_end_time, latest_end_time, event_type):
Xinan Linea1efcb2019-12-30 23:46:42 -0800451 """Get passed builds inside a given time span.
Xixuan Wuf856ff12019-05-21 14:09:38 -0700452
Xinan Lin3330d672020-03-03 14:52:36 -0800453 BigQuery does not guarantee the inserted time of rows. A new build
454 may not get inserted when suite scheduler runs the query. To avoid
Xinan Lin71eeeb02020-03-10 17:37:12 -0700455 it, we scan each time span twice:
Xinan Lin3330d672020-03-03 14:52:36 -0800456 - the first run catches the new build from earliest_end_time to
457 latest_end_time, and inserts the result to a temp BQ table.
458 - the second run checks the build from (earliest_end_time - 1Day)
459 to (latest_end_time - 1Day) plus (earliest_end_time to
460 latest_end_time). The query returns the build which does not
461 appear in the temp table. Thus, if a build was not fetched by the
Xinan Lin71eeeb02020-03-10 17:37:12 -0700462 first run, we still could schedule test on it at most 1 day later
Xinan Lin2bbd9d32020-04-03 18:03:34 -0700463 for new_build events.
464 Weekly and nightly events do not need this arrangement because
465 they do not cover every single build.
466
Xinan Lin3330d672020-03-03 14:52:36 -0800467
Xixuan Wuf856ff12019-05-21 14:09:38 -0700468 Args:
Xinan Linea1efcb2019-12-30 23:46:42 -0800469 earliest_end_time: a datetime.datetime object in UTC.
470 latest_end_time: a datetime.datetime object in UTC.
Xinan Lin71eeeb02020-03-10 17:37:12 -0700471 event_type: a string of event type. It could be one of
472 [WEEKLY|NIGHTLY|new_build].
Xixuan Wuf856ff12019-05-21 14:09:38 -0700473
474 Returns:
475 A list of build_lib.BuildInfo objects.
476 """
Xinan Lin761b0c52020-03-25 17:31:57 -0700477 base_query_template = """
Sean McAllister53dd3d82021-05-18 15:15:14 -0600478 WITH builds AS
479 (SELECT
480 COALESCE(
481 JSON_EXTRACT_SCALAR(output.properties, '$.board'),
482 JSON_EXTRACT_SCALAR(output.properties, '$.build_targets[0].name')
483 ) AS board,
484 COALESCE(
485 JSON_EXTRACT_SCALAR(output.properties, '$.milestone_version'),
486 REPLACE(
487 SPLIT(JSON_EXTRACT_SCALAR(output.properties, '$.full_version'), '-')[OFFSET(0)], 'R', '')
488 ) AS milestone,
489 COALESCE(
490 JSON_EXTRACT_SCALAR(output.properties, '$.platform_version'),
491 SPLIT(JSON_EXTRACT_SCALAR(output.properties, '$.full_version'), '-')[OFFSET(1)]
492 ) AS platform,
493 COALESCE(
494 JSON_EXTRACT_SCALAR(output.properties, '$.cbb_config'),
495 builder.builder
496 ) AS build_config,
497
498 -- Time info
499 end_time as build_end_time,
500 CURRENT_TIMESTAMP() as inserted_time,
501 FROM `cr-buildbucket.chromeos.builds`
502 WHERE
503 status = 'SUCCESS'
504 AND JSON_EXTRACT_SCALAR(output.properties, '$.suite_scheduling') = 'True')
505
506 SELECT
507 *,
508 '{0}' as event_type
509 FROM builds
510 WHERE
511 board IS NOT NULL
Xixuan Wuf856ff12019-05-21 14:09:38 -0700512 """
Xinan Lin761b0c52020-03-25 17:31:57 -0700513 base_query_str = base_query_template.format(event_type)
Xinan Lin71eeeb02020-03-10 17:37:12 -0700514 earliest_end_time_str = earliest_end_time.strftime(
515 time_converter.TIME_FORMAT)
516 latest_end_time_str = latest_end_time.strftime(time_converter.TIME_FORMAT)
517 project_id = constants.AppID.STAGING_APP
518 if constants.environment() == constants.RunningEnv.ENV_PROD:
519 project_id = constants.application_id()
Xinan Lin3330d672020-03-03 14:52:36 -0800520
Xinan Lin2bbd9d32020-04-03 18:03:34 -0700521 if event_type == 'new_build':
522 insert_passed_builds = """
523 INSERT
524 `google.com:{0}.builds.passed_builds`(
525 board,
526 milestone,
527 platform,
528 build_config,
529 build_end_time,
530 inserted_time,
531 event_type
532 ) {1}
533 AND end_time > '{2}'
534 AND end_time < '{3}'
535 """
536 # Insert the currently visible builds to BQ.
537 logging.info(
538 'Insert the visible passed builds '
539 'between %s and %s to BQ.', earliest_end_time_str,
540 latest_end_time_str)
541 self.query(
542 insert_passed_builds.format(project_id, base_query_str,
543 earliest_end_time_str,
544 latest_end_time_str))
Xixuan Wuf856ff12019-05-21 14:09:38 -0700545
Sean McAllister53dd3d82021-05-18 15:15:14 -0600546 query_template = _passed_build_query_template(event_type)
Xinan Lin2bbd9d32020-04-03 18:03:34 -0700547 query_str = query_template.format(base_query_str, earliest_end_time_str,
548 latest_end_time_str, project_id)
Xinan Lin3330d672020-03-03 14:52:36 -0800549 if global_config.GAE_TESTING:
Xinan Lin71eeeb02020-03-10 17:37:12 -0700550 query_str += 'LIMIT 10'
551 logging.info('Getting passed builds finished between %s and %s',
552 earliest_end_time_str, latest_end_time_str)
Xinan Lin2bbd9d32020-04-03 18:03:34 -0700553 res = self.query(query_str)
Xixuan Wuf856ff12019-05-21 14:09:38 -0700554 res = _parse_bq_job_query(res)
555 if res is None:
556 return []
557
558 build_infos = []
559 for board, milestone, platform, build_config in res:
Xixuan Wuf856ff12019-05-21 14:09:38 -0700560 build_infos.append(
561 build_lib.BuildInfo(board, None, milestone, platform, build_config))
562
563 return build_infos
564
Xinan Lin71eeeb02020-03-10 17:37:12 -0700565 def get_relaxed_passed_builds(self, earliest_end_time, latest_end_time,
566 event_type):
Sean McAllister53dd3d82021-05-18 15:15:14 -0600567 """Get builds with successful SkylabHWTest stages between a given span.
Xixuan Wuf856ff12019-05-21 14:09:38 -0700568
Xinan Lin71eeeb02020-03-10 17:37:12 -0700569 Same as get_passed_builds, we run the query twice to ensure we
Xinan Lin2bbd9d32020-04-03 18:03:34 -0700570 fetched all builds from BQ for new_build event.
Xinan Lin3330d672020-03-03 14:52:36 -0800571
Xixuan Wuf856ff12019-05-21 14:09:38 -0700572 Args:
Xinan Linea1efcb2019-12-30 23:46:42 -0800573 earliest_end_time: a datetime.datetime object in UTC.
574 latest_end_time: a datetime.datetime object in UTC.
Xinan Lin71eeeb02020-03-10 17:37:12 -0700575 event_type: a string of event type.
Xixuan Wuf856ff12019-05-21 14:09:38 -0700576
577 Returns:
578 A list of build_lib.BuildInfo objects.
579 """
Xinan Lin761b0c52020-03-25 17:31:57 -0700580 base_query_template = """
Sean McAllister53dd3d82021-05-18 15:15:14 -0600581 WITH builds AS
582 (SELECT
583 COALESCE(
584 JSON_EXTRACT_SCALAR(output.properties, '$.board'),
585 JSON_EXTRACT_SCALAR(output.properties, '$.build_targets[0].name')
586 ) AS board,
587 COALESCE(
588 JSON_EXTRACT_SCALAR(output.properties, '$.milestone_version'),
589 REPLACE(
590 SPLIT(JSON_EXTRACT_SCALAR(output.properties, '$.full_version'), '-')[OFFSET(0)], 'R', '')
591 ) AS milestone,
592 COALESCE(
593 JSON_EXTRACT_SCALAR(output.properties, '$.platform_version'),
594 SPLIT(JSON_EXTRACT_SCALAR(output.properties, '$.full_version'), '-')[OFFSET(1)]
595 ) AS platform,
596 COALESCE(
597 JSON_EXTRACT_SCALAR(output.properties, '$.cbb_config'),
598 builder.builder
599 ) AS build_config,
600
601 step.name AS stage_name,
602
603 -- Time info
604 build.end_time as build_end_time,
605 CURRENT_TIMESTAMP() as inserted_time,
606 FROM `cr-buildbucket.chromeos.builds` build,
607 UNNEST(build.steps) AS step
608 WHERE
609 build.status != 'SUCCESS'
610 AND step.name like 'SkylabHWTest%%'
611 AND step.status = 'SUCCESS'
612 AND JSON_EXTRACT_SCALAR(output.properties, '$.suite_scheduling') = 'True')
613
614 SELECT
615 *,
616 '{0}' AS event_type
617 FROM
618 builds
619 WHERE board IS NOT NULL
Xixuan Wuf856ff12019-05-21 14:09:38 -0700620 """
Sean McAllister53dd3d82021-05-18 15:15:14 -0600621
Xinan Lin761b0c52020-03-25 17:31:57 -0700622 base_query_str = base_query_template.format(event_type)
Xinan Lin71eeeb02020-03-10 17:37:12 -0700623 earliest_end_time_str = earliest_end_time.strftime(
624 time_converter.TIME_FORMAT)
625 latest_end_time_str = latest_end_time.strftime(time_converter.TIME_FORMAT)
626 project_id = constants.AppID.STAGING_APP
627 if constants.environment() == constants.RunningEnv.ENV_PROD:
628 project_id = constants.application_id()
Xinan Lin3330d672020-03-03 14:52:36 -0800629
Xinan Lin2bbd9d32020-04-03 18:03:34 -0700630 if event_type == 'new_build':
631 insert_relaxed_builds = """
632 INSERT
633 `google.com:{0}.builds.relaxed_builds`(
Xinan Lin2bbd9d32020-04-03 18:03:34 -0700634 stage_name,
635 board,
636 milestone,
637 platform,
638 build_config,
639 build_end_time,
640 inserted_time,
641 event_type
642 ) {1}
643 AND c.end_time > '{2}'
644 AND c.end_time < '{3}'
645 """
646 logging.info(
647 'Insert the visible relaxed builds '
648 'between %s and %s to BQ.', earliest_end_time_str,
649 latest_end_time_str)
650 self.query(
651 insert_relaxed_builds.format(project_id, base_query_str,
652 earliest_end_time_str,
653 latest_end_time_str))
Xixuan Wuf856ff12019-05-21 14:09:38 -0700654
Sean McAllister53dd3d82021-05-18 15:15:14 -0600655 query_template = _relaxed_build_query_template(event_type)
Xinan Lin2bbd9d32020-04-03 18:03:34 -0700656 query_str = query_template.format(base_query_str, earliest_end_time_str,
657 latest_end_time_str, project_id)
Xinan Lin3330d672020-03-03 14:52:36 -0800658 if global_config.GAE_TESTING:
Xinan Lin71eeeb02020-03-10 17:37:12 -0700659 query_str += 'LIMIT 10'
660 logging.info('Getting relaxed passed builds finished between %s and %s',
661 earliest_end_time_str, latest_end_time_str)
Xinan Lin2bbd9d32020-04-03 18:03:34 -0700662 res = self.query(query_str)
Xixuan Wuf856ff12019-05-21 14:09:38 -0700663 res = _parse_bq_job_query(res)
664 if res is None:
665 return []
666
667 build_infos = []
Sean McAllister53dd3d82021-05-18 15:15:14 -0600668 for stage_name, board, milestone, platform, build_config in res:
669 model = _try_parse_model(stage_name)
Xixuan Wuf856ff12019-05-21 14:09:38 -0700670 build_infos.append(
671 build_lib.BuildInfo(board, model, milestone, platform, build_config))
672
673 return build_infos
674
675
Xixuan Wu55d38c52019-05-21 14:26:23 -0700676def _parse_bq_job_query(json_input):
677 """Parse response from API bigquery.jobs.query.
678
679 Args:
680 json_input: a dict, representing jsons returned by query API.
681
682 Returns:
683 A 2D string matrix: [rows[columns]], or None if no result.
684 E.g. Input:
685 "rows": [
686 {
687 "f": [ # field
688 {
689 "v": 'foo1',
690 },
691 {
692 "v": 'foo2',
693 }
694 ]
695 }
696 {
697 "f": [ # field
698 {
699 "v": 'bar1',
700 },
701 {
702 "v": 'bar2',
703 }
704 ]
705 }
706 ]
707 => Output: [['foo1', 'foo2'], ['bar1', 'bar2']]
708 """
709 if 'rows' not in json_input:
710 return None
711
712 res = []
713 for r in json_input['rows']:
714 rc = []
715 for c in r['f']:
716 rc.append(c['v'])
717
718 res.append(rc)
719
720 return res
Xixuan Wuf856ff12019-05-21 14:09:38 -0700721
722
Sean McAllister53dd3d82021-05-18 15:15:14 -0600723def _try_parse_model(build_stage_name):
724 """Try to parse model name from the SkylabHWTest stage name.
Xixuan Wuf856ff12019-05-21 14:09:38 -0700725
Sean McAllister53dd3d82021-05-18 15:15:14 -0600726 An example build_stage_name is 'HWTest [bvt-isntaller] [whitetip]'. So we'll
727 regexp match on this to get the board name out of the second bracket.
728
Xixuan Wuf856ff12019-05-21 14:09:38 -0700729 Args:
730 build_stage_name: The stage name of a HWTest sanity stage, e.g.
Sean McAllister53dd3d82021-05-18 15:15:14 -0600731 "SkylabHWTest [bvt-installer] [whitetip]".
Xixuan Wuf856ff12019-05-21 14:09:38 -0700732
733 Returns:
Sean McAllister53dd3d82021-05-18 15:15:14 -0600734 A model name, e.g. "whitetip" or None if not found.
Xixuan Wuf856ff12019-05-21 14:09:38 -0700735 """
Xixuan Wuf856ff12019-05-21 14:09:38 -0700736
Sean McAllister53dd3d82021-05-18 15:15:14 -0600737 match = re.search(
738 "SkylabHWTest \[[a-z0-9-]+\] \[([a-z0-9-]+)\]",
739 build_stage_name,
740 )
Xixuan Wuf856ff12019-05-21 14:09:38 -0700741
Sean McAllister53dd3d82021-05-18 15:15:14 -0600742 if match:
743 return match[1]
744 return None
Xixuan Wuf856ff12019-05-21 14:09:38 -0700745
Sean McAllister53dd3d82021-05-18 15:15:14 -0600746def _passed_build_query_template(event_type):
Xinan Lin2bbd9d32020-04-03 18:03:34 -0700747 """Wrapper to create the query template for passed builds."""
748 if event_type == 'new_build':
749 return """
750 WITH passed_builds AS
751 (
752 {0}
753 AND (
754 (end_time > TIMESTAMP_SUB(
755 PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S', '{1}'), INTERVAL 1 DAY)
756 AND end_time < TIMESTAMP_SUB(
757 PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S', '{2}'), INTERVAL 1 DAY))
758 OR (end_time > '{1}' AND end_time < '{2}')
759 )
760 )
761 SELECT
762 b.board,
763 b.milestone,
764 b.platform,
765 b.build_config,
766 FROM
767 passed_builds AS b
768 LEFT JOIN
769 `google.com:{3}.builds.passed_builds` AS r
770 ON (
771 r.board = b.board
772 AND r.milestone = b.milestone
773 AND r.build_config = b.build_config
774 AND r.platform = b.platform
775 AND r.event_type = b.event_type
776 AND r.build_end_time > TIMESTAMP_SUB(
777 PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S', '{1}'),
778 INTERVAL 1 DAY)
779 AND r.build_end_time < TIMESTAMP_SUB(
780 PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S', '{2}'),
781 INTERVAL 1 DAY)
782 )
783 WHERE
784 r.inserted_time is null
785 """
786 return """
787 WITH passed_builds AS
788 (
789 {0}
790 AND end_time > '{1}'
791 AND end_time < '{2}'
792 )
793 SELECT
794 b.board,
795 b.milestone,
796 b.platform,
797 b.build_config,
798 FROM
799 passed_builds AS b
800 """
801
802
Sean McAllister53dd3d82021-05-18 15:15:14 -0600803def _relaxed_build_query_template(event_type):
Xinan Lin2bbd9d32020-04-03 18:03:34 -0700804 """Wrapper to create the query template for relaxed builds."""
805 if event_type == 'new_build':
806 return """
807 WITH relaxed_builds AS
808 (
809 {0}
810 AND (
811 (c.end_time > TIMESTAMP_SUB(
812 PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S', '{1}'), INTERVAL 1 DAY)
813 AND c.end_time < TIMESTAMP_SUB(
814 PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S', '{2}'), INTERVAL 1 DAY))
815 OR (c.end_time > '{1}' AND c.end_time < '{2}')
816 )
817 )
818 SELECT
Xinan Lin2bbd9d32020-04-03 18:03:34 -0700819 b.stage_name,
820 b.board,
821 b.milestone,
822 b.platform,
823 b.build_config,
824 FROM
825 relaxed_builds AS b
826 LEFT JOIN
827 `google.com:{3}.builds.relaxed_builds` AS r
828 ON (
829 r.board = b.board
830 AND r.milestone = b.milestone
831 AND r.build_config = b.build_config
832 AND r.platform = b.platform
833 AND r.event_type = b.event_type
834 AND r.build_end_time > TIMESTAMP_SUB(
835 PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S', '{1}'),
836 INTERVAL 1 DAY)
837 AND r.build_end_time < TIMESTAMP_SUB(
838 PARSE_TIMESTAMP('%Y-%m-%d %H:%M:%S', '{2}'),
839 INTERVAL 1 DAY)
840 )
841 WHERE
842 r.inserted_time is null
843 """
844 return """
845 WITH relaxed_builds AS
846 (
847 {0}
848 AND c.end_time > '{1}'
849 AND c.end_time < '{2}'
850 )
851 SELECT
Xinan Lin2bbd9d32020-04-03 18:03:34 -0700852 b.stage_name,
853 b.board,
854 b.milestone,
855 b.platform,
856 b.build_config,
857 FROM
858 relaxed_builds AS b
859 """