blob: 63a91fba9dd6000f087ff875f8ca849e9b1d8c28 [file] [log] [blame]
szager@chromium.orgb4696232013-10-16 19:45:35 +00001# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""
6Utilities for requesting information for a gerrit server via https.
7
8https://gerrit-review.googlesource.com/Documentation/rest-api.html
9"""
10
11import base64
Dan Jacques8d11e482016-11-15 14:25:56 -080012import contextlib
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +000013import cookielib
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +010014import httplib # Still used for its constants.
szager@chromium.orgb4696232013-10-16 19:45:35 +000015import json
16import logging
17import netrc
18import os
nodir@chromium.orgce32b6e2014-05-12 20:31:32 +000019import re
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +000020import socket
szager@chromium.orgf202a252014-05-27 18:55:52 +000021import stat
22import sys
Dan Jacques8d11e482016-11-15 14:25:56 -080023import tempfile
szager@chromium.orgb4696232013-10-16 19:45:35 +000024import time
25import urllib
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +000026import urlparse
szager@chromium.orgb4696232013-10-16 19:45:35 +000027from cStringIO import StringIO
28
Dan Jacques8d11e482016-11-15 14:25:56 -080029import gclient_utils
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +010030from third_party import httplib2
szager@chromium.orgf202a252014-05-27 18:55:52 +000031
szager@chromium.orgb4696232013-10-16 19:45:35 +000032LOGGER = logging.getLogger()
33TRY_LIMIT = 5
34
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +000035
szager@chromium.orgb4696232013-10-16 19:45:35 +000036# Controls the transport protocol used to communicate with gerrit.
37# This is parameterized primarily to enable GerritTestCase.
38GERRIT_PROTOCOL = 'https'
39
40
41class GerritError(Exception):
42 """Exception class for errors commuicating with the gerrit-on-borg service."""
43 def __init__(self, http_status, *args, **kwargs):
44 super(GerritError, self).__init__(*args, **kwargs)
45 self.http_status = http_status
46 self.message = '(%d) %s' % (self.http_status, self.message)
47
48
nodir@chromium.orgce32b6e2014-05-12 20:31:32 +000049class GerritAuthenticationError(GerritError):
50 """Exception class for authentication errors during Gerrit communication."""
51
52
szager@chromium.orgb4696232013-10-16 19:45:35 +000053def _QueryString(param_dict, first_param=None):
54 """Encodes query parameters in the key:val[+key:val...] format specified here:
55
56 https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#list-changes
57 """
58 q = [urllib.quote(first_param)] if first_param else []
59 q.extend(['%s:%s' % (key, val) for key, val in param_dict.iteritems()])
60 return '+'.join(q)
61
62
63def GetConnectionClass(protocol=None):
64 if protocol is None:
65 protocol = GERRIT_PROTOCOL
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +010066 if protocol in ('http', 'https'):
67 return httplib2.Http
szager@chromium.orgb4696232013-10-16 19:45:35 +000068 else:
69 raise RuntimeError(
70 "Don't know how to work with protocol '%s'" % protocol)
71
72
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +000073class Authenticator(object):
74 """Base authenticator class for authenticator implementations to subclass."""
75
76 def get_auth_header(self, host):
77 raise NotImplementedError()
78
79 @staticmethod
80 def get():
81 """Returns: (Authenticator) The identified Authenticator to use.
82
83 Probes the local system and its environment and identifies the
84 Authenticator instance to use.
85 """
86 if GceAuthenticator.is_gce():
87 return GceAuthenticator()
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +000088 return CookiesAuthenticator()
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +000089
90
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +000091class CookiesAuthenticator(Authenticator):
92 """Authenticator implementation that uses ".netrc" or ".gitcookies" for token.
93
94 Expected case for developer workstations.
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +000095 """
96
97 def __init__(self):
98 self.netrc = self._get_netrc()
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +000099 self.gitcookies = self._get_gitcookies()
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000100
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000101 @classmethod
102 def get_new_password_message(cls, host):
103 assert not host.startswith('http')
104 # Assume *.googlesource.com pattern.
105 parts = host.split('.')
106 if not parts[0].endswith('-review'):
107 parts[0] += '-review'
108 url = 'https://%s/new-password' % ('.'.join(parts))
Andrii Shyshkalov0a0b0672017-03-16 16:27:48 +0100109 return 'You can (re)generate your credentials by visiting %s' % url
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000110
111 @classmethod
112 def get_netrc_path(cls):
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000113 path = '_netrc' if sys.platform.startswith('win') else '.netrc'
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000114 return os.path.expanduser(os.path.join('~', path))
115
116 @classmethod
117 def _get_netrc(cls):
Dan Jacques8d11e482016-11-15 14:25:56 -0800118 # Buffer the '.netrc' path. Use an empty file if it doesn't exist.
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000119 path = cls.get_netrc_path()
Dan Jacques8d11e482016-11-15 14:25:56 -0800120 content = ''
121 if os.path.exists(path):
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000122 st = os.stat(path)
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000123 if st.st_mode & (stat.S_IRWXG | stat.S_IRWXO):
124 print >> sys.stderr, (
125 'WARNING: netrc file %s cannot be used because its file '
126 'permissions are insecure. netrc file permissions should be '
127 '600.' % path)
Dan Jacques8d11e482016-11-15 14:25:56 -0800128 with open(path) as fd:
129 content = fd.read()
130
131 # Load the '.netrc' file. We strip comments from it because processing them
132 # can trigger a bug in Windows. See crbug.com/664664.
133 content = '\n'.join(l for l in content.splitlines()
134 if l.strip() and not l.strip().startswith('#'))
135 with tempdir() as tdir:
136 netrc_path = os.path.join(tdir, 'netrc')
137 with open(netrc_path, 'w') as fd:
138 fd.write(content)
139 os.chmod(netrc_path, (stat.S_IRUSR | stat.S_IWUSR))
140 return cls._get_netrc_from_path(netrc_path)
141
142 @classmethod
143 def _get_netrc_from_path(cls, path):
144 try:
145 return netrc.netrc(path)
146 except IOError:
147 print >> sys.stderr, 'WARNING: Could not read netrc file %s' % path
148 return netrc.netrc(os.devnull)
149 except netrc.NetrcParseError as e:
150 print >> sys.stderr, ('ERROR: Cannot use netrc file %s due to a '
151 'parsing error: %s' % (path, e))
152 return netrc.netrc(os.devnull)
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000153
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000154 @classmethod
155 def get_gitcookies_path(cls):
Ravi Mistry0bfa9ad2016-11-21 12:58:31 -0500156 if os.getenv('GIT_COOKIES_PATH'):
157 return os.getenv('GIT_COOKIES_PATH')
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000158 return os.path.join(os.environ['HOME'], '.gitcookies')
159
160 @classmethod
161 def _get_gitcookies(cls):
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +0000162 gitcookies = {}
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000163 path = cls.get_gitcookies_path()
164 if not os.path.exists(path):
165 return gitcookies
166
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +0000167 try:
168 f = open(path, 'rb')
169 except IOError:
170 return gitcookies
171
172 with f:
173 for line in f:
174 try:
175 fields = line.strip().split('\t')
176 if line.strip().startswith('#') or len(fields) != 7:
177 continue
178 domain, xpath, key, value = fields[0], fields[2], fields[5], fields[6]
179 if xpath == '/' and key == 'o':
180 login, secret_token = value.split('=', 1)
181 gitcookies[domain] = (login, secret_token)
182 except (IndexError, ValueError, TypeError) as exc:
Andrii Shyshkalov5b04a572017-01-23 17:44:41 +0100183 LOGGER.warning(exc)
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +0000184
185 return gitcookies
186
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000187 def get_auth_header(self, host):
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +0000188 auth = None
189 for domain, creds in self.gitcookies.iteritems():
190 if cookielib.domain_match(host, domain):
191 auth = (creds[0], None, creds[1])
192 break
193
194 if not auth:
195 auth = self.netrc.authenticators(host)
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000196 if auth:
197 return 'Basic %s' % (base64.b64encode('%s:%s' % (auth[0], auth[2])))
198 return None
199
Andrii Shyshkalov18975322017-01-25 16:44:13 +0100200
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000201# Backwards compatibility just in case somebody imports this outside of
202# depot_tools.
203NetrcAuthenticator = CookiesAuthenticator
204
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000205
206class GceAuthenticator(Authenticator):
207 """Authenticator implementation that uses GCE metadata service for token.
208 """
209
210 _INFO_URL = 'http://metadata.google.internal'
211 _ACQUIRE_URL = ('http://metadata/computeMetadata/v1/instance/'
212 'service-accounts/default/token')
213 _ACQUIRE_HEADERS = {"Metadata-Flavor": "Google"}
214
215 _cache_is_gce = None
216 _token_cache = None
217 _token_expiration = None
218
219 @classmethod
220 def is_gce(cls):
Ravi Mistryfad941b2016-11-15 13:00:47 -0500221 if os.getenv('SKIP_GCE_AUTH_FOR_GIT'):
222 return False
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000223 if cls._cache_is_gce is None:
224 cls._cache_is_gce = cls._test_is_gce()
225 return cls._cache_is_gce
226
227 @classmethod
228 def _test_is_gce(cls):
229 # Based on https://cloud.google.com/compute/docs/metadata#runninggce
230 try:
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100231 resp, _ = cls._get(cls._INFO_URL)
232 except (socket.error, httplib2.ServerNotFoundError):
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000233 # Could not resolve URL.
234 return False
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100235 return resp.get('metadata-flavor') == 'Google'
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000236
237 @staticmethod
238 def _get(url, **kwargs):
239 next_delay_sec = 1
240 for i in xrange(TRY_LIMIT):
241 if i > 0:
242 # Retry server error status codes.
243 LOGGER.info('Encountered server error; retrying after %d second(s).',
244 next_delay_sec)
245 time.sleep(next_delay_sec)
246 next_delay_sec *= 2
247
248 p = urlparse.urlparse(url)
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100249 c = GetConnectionClass(protocol=p.scheme)()
250 resp, contents = c.request(url, 'GET', **kwargs)
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000251 LOGGER.debug('GET [%s] #%d/%d (%d)', url, i+1, TRY_LIMIT, resp.status)
252 if resp.status < httplib.INTERNAL_SERVER_ERROR:
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100253 return (resp, contents)
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000254
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000255 @classmethod
256 def _get_token_dict(cls):
257 if cls._token_cache:
258 # If it expires within 25 seconds, refresh.
259 if cls._token_expiration < time.time() - 25:
260 return cls._token_cache
261
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100262 resp, contents = cls._get(cls._ACQUIRE_URL, headers=cls._ACQUIRE_HEADERS)
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000263 if resp.status != httplib.OK:
264 return None
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100265 cls._token_cache = json.loads(contents)
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000266 cls._token_expiration = cls._token_cache['expires_in'] + time.time()
267 return cls._token_cache
268
269 def get_auth_header(self, _host):
270 token_dict = self._get_token_dict()
271 if not token_dict:
272 return None
273 return '%(token_type)s %(access_token)s' % token_dict
274
275
szager@chromium.orgb4696232013-10-16 19:45:35 +0000276def CreateHttpConn(host, path, reqtype='GET', headers=None, body=None):
277 """Opens an https connection to a gerrit service, and sends a request."""
278 headers = headers or {}
279 bare_host = host.partition(':')[0]
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000280
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000281 auth = Authenticator.get().get_auth_header(bare_host)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000282 if auth:
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000283 headers.setdefault('Authorization', auth)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000284 else:
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000285 LOGGER.debug('No authorization found for %s.' % bare_host)
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000286
Dan Jacques6d5bcc22016-11-14 15:32:04 -0800287 url = path
288 if not url.startswith('/'):
289 url = '/' + url
290 if 'Authorization' in headers and not url.startswith('/a/'):
291 url = '/a%s' % url
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000292
szager@chromium.orgb4696232013-10-16 19:45:35 +0000293 if body:
294 body = json.JSONEncoder().encode(body)
295 headers.setdefault('Content-Type', 'application/json')
296 if LOGGER.isEnabledFor(logging.DEBUG):
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000297 LOGGER.debug('%s %s://%s%s' % (reqtype, GERRIT_PROTOCOL, host, url))
szager@chromium.orgb4696232013-10-16 19:45:35 +0000298 for key, val in headers.iteritems():
299 if key == 'Authorization':
300 val = 'HIDDEN'
301 LOGGER.debug('%s: %s' % (key, val))
302 if body:
303 LOGGER.debug(body)
304 conn = GetConnectionClass()(host)
305 conn.req_host = host
306 conn.req_params = {
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100307 'uri': urlparse.urljoin('%s://%s' % (GERRIT_PROTOCOL, host), url),
szager@chromium.orgb4696232013-10-16 19:45:35 +0000308 'method': reqtype,
309 'headers': headers,
310 'body': body,
311 }
szager@chromium.orgb4696232013-10-16 19:45:35 +0000312 return conn
313
314
315def ReadHttpResponse(conn, expect_status=200, ignore_404=True):
316 """Reads an http response from a connection into a string buffer.
317
318 Args:
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100319 conn: An Http object created by CreateHttpConn above.
szager@chromium.orgb4696232013-10-16 19:45:35 +0000320 expect_status: Success is indicated by this status in the response.
321 ignore_404: For many requests, gerrit-on-borg will return 404 if the request
322 doesn't match the database contents. In most such cases, we
323 want the API to return None rather than raise an Exception.
324 Returns: A string buffer containing the connection's reply.
325 """
326
327 sleep_time = 0.5
328 for idx in range(TRY_LIMIT):
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100329 response, contents = conn.request(**conn.req_params)
nodir@chromium.orgce32b6e2014-05-12 20:31:32 +0000330
331 # Check if this is an authentication issue.
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100332 www_authenticate = response.get('www-authenticate')
nodir@chromium.orgce32b6e2014-05-12 20:31:32 +0000333 if (response.status in (httplib.UNAUTHORIZED, httplib.FOUND) and
334 www_authenticate):
335 auth_match = re.search('realm="([^"]+)"', www_authenticate, re.I)
336 host = auth_match.group(1) if auth_match else conn.req_host
337 reason = ('Authentication failed. Please make sure your .netrc file '
338 'has credentials for %s' % host)
339 raise GerritAuthenticationError(response.status, reason)
340
szager@chromium.orgb4696232013-10-16 19:45:35 +0000341 # If response.status < 500 then the result is final; break retry loop.
342 if response.status < 500:
Andrii Shyshkalov5b04a572017-01-23 17:44:41 +0100343 LOGGER.debug('got response %d for %s %s', response.status,
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100344 conn.req_params['method'], conn.req_params['uri'])
szager@chromium.orgb4696232013-10-16 19:45:35 +0000345 break
346 # A status >=500 is assumed to be a possible transient error; retry.
347 http_version = 'HTTP/%s' % ('1.1' if response.version == 11 else '1.0')
Andrii Shyshkalov5b04a572017-01-23 17:44:41 +0100348 LOGGER.warn('A transient error occurred while querying %s:\n'
349 '%s %s %s\n'
350 '%s %d %s',
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100351 conn.host, conn.req_params['method'], conn.req_params['uri'],
Andrii Shyshkalov5b04a572017-01-23 17:44:41 +0100352 http_version, http_version, response.status, response.reason)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000353 if TRY_LIMIT - idx > 1:
Andrii Shyshkalov5b04a572017-01-23 17:44:41 +0100354 LOGGER.warn('... will retry %d more times.', TRY_LIMIT - idx - 1)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000355 time.sleep(sleep_time)
356 sleep_time = sleep_time * 2
szager@chromium.orgb4696232013-10-16 19:45:35 +0000357 if ignore_404 and response.status == 404:
358 return StringIO()
359 if response.status != expect_status:
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100360 reason = '%s: %s' % (response.reason, contents)
nodir@chromium.orga7798032014-04-30 23:40:53 +0000361 raise GerritError(response.status, reason)
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100362 return StringIO(contents)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000363
364
365def ReadHttpJsonResponse(conn, expect_status=200, ignore_404=True):
366 """Parses an https response as json."""
367 fh = ReadHttpResponse(
368 conn, expect_status=expect_status, ignore_404=ignore_404)
369 # The first line of the response should always be: )]}'
370 s = fh.readline()
371 if s and s.rstrip() != ")]}'":
372 raise GerritError(200, 'Unexpected json output: %s' % s)
373 s = fh.read()
374 if not s:
375 return None
376 return json.loads(s)
377
378
379def QueryChanges(host, param_dict, first_param=None, limit=None, o_params=None,
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100380 start=None):
szager@chromium.orgb4696232013-10-16 19:45:35 +0000381 """
382 Queries a gerrit-on-borg server for changes matching query terms.
383
384 Args:
385 param_dict: A dictionary of search parameters, as documented here:
386 http://gerrit-documentation.googlecode.com/svn/Documentation/2.6/user-search.html
387 first_param: A change identifier
388 limit: Maximum number of results to return.
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100389 start: how many changes to skip (starting with the most recent)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000390 o_params: A list of additional output specifiers, as documented here:
391 https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#list-changes
392 Returns:
393 A list of json-decoded query results.
394 """
395 # Note that no attempt is made to escape special characters; YMMV.
396 if not param_dict and not first_param:
397 raise RuntimeError('QueryChanges requires search parameters')
398 path = 'changes/?q=%s' % _QueryString(param_dict, first_param)
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100399 if start:
400 path = '%s&start=%s' % (path, start)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000401 if limit:
402 path = '%s&n=%d' % (path, limit)
403 if o_params:
404 path = '%s&%s' % (path, '&'.join(['o=%s' % p for p in o_params]))
405 # Don't ignore 404; a query should always return a list, even if it's empty.
406 return ReadHttpJsonResponse(CreateHttpConn(host, path), ignore_404=False)
407
408
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000409def GenerateAllChanges(host, param_dict, first_param=None, limit=500,
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100410 o_params=None, start=None):
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000411 """
412 Queries a gerrit-on-borg server for all the changes matching the query terms.
413
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100414 WARNING: this is unreliable if a change matching the query is modified while
415 this function is being called.
416
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000417 A single query to gerrit-on-borg is limited on the number of results by the
418 limit parameter on the request (see QueryChanges) and the server maximum
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100419 limit.
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000420
421 Args:
422 param_dict, first_param: Refer to QueryChanges().
423 limit: Maximum number of requested changes per query.
424 o_params: Refer to QueryChanges().
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100425 start: Refer to QueryChanges().
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000426
427 Returns:
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100428 A generator object to the list of returned changes.
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000429 """
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100430 already_returned = set()
431 def at_most_once(cls):
432 for cl in cls:
433 if cl['_number'] not in already_returned:
434 already_returned.add(cl['_number'])
435 yield cl
436
437 start = start or 0
438 cur_start = start
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000439 more_changes = True
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100440
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000441 while more_changes:
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100442 # This will fetch changes[start..start+limit] sorted by most recently
443 # updated. Since the rank of any change in this list can be changed any time
444 # (say user posting comment), subsequent calls may overalp like this:
445 # > initial order ABCDEFGH
446 # query[0..3] => ABC
447 # > E get's updated. New order: EABCDFGH
448 # query[3..6] => CDF # C is a dup
449 # query[6..9] => GH # E is missed.
450 page = QueryChanges(host, param_dict, first_param, limit, o_params,
451 cur_start)
452 for cl in at_most_once(page):
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000453 yield cl
454
455 more_changes = [cl for cl in page if '_more_changes' in cl]
456 if len(more_changes) > 1:
457 raise GerritError(
458 200,
459 'Received %d changes with a _more_changes attribute set but should '
460 'receive at most one.' % len(more_changes))
461 if more_changes:
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100462 cur_start += len(page)
463
464 # If we paged through, query again the first page which in most circumstances
465 # will fetch all changes that were modified while this function was run.
466 if start != cur_start:
467 page = QueryChanges(host, param_dict, first_param, limit, o_params, start)
468 for cl in at_most_once(page):
469 yield cl
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000470
471
szager@chromium.orgb4696232013-10-16 19:45:35 +0000472def MultiQueryChanges(host, param_dict, change_list, limit=None, o_params=None,
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100473 start=None):
szager@chromium.orgb4696232013-10-16 19:45:35 +0000474 """Initiate a query composed of multiple sets of query parameters."""
475 if not change_list:
476 raise RuntimeError(
477 "MultiQueryChanges requires a list of change numbers/id's")
478 q = ['q=%s' % '+OR+'.join([urllib.quote(str(x)) for x in change_list])]
479 if param_dict:
480 q.append(_QueryString(param_dict))
481 if limit:
482 q.append('n=%d' % limit)
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100483 if start:
484 q.append('S=%s' % start)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000485 if o_params:
486 q.extend(['o=%s' % p for p in o_params])
487 path = 'changes/?%s' % '&'.join(q)
488 try:
489 result = ReadHttpJsonResponse(CreateHttpConn(host, path), ignore_404=False)
490 except GerritError as e:
491 msg = '%s:\n%s' % (e.message, path)
492 raise GerritError(e.http_status, msg)
493 return result
494
495
496def GetGerritFetchUrl(host):
497 """Given a gerrit host name returns URL of a gerrit instance to fetch from."""
498 return '%s://%s/' % (GERRIT_PROTOCOL, host)
499
500
501def GetChangePageUrl(host, change_number):
502 """Given a gerrit host name and change number, return change page url."""
503 return '%s://%s/#/c/%d/' % (GERRIT_PROTOCOL, host, change_number)
504
505
506def GetChangeUrl(host, change):
507 """Given a gerrit host name and change id, return an url for the change."""
508 return '%s://%s/a/changes/%s' % (GERRIT_PROTOCOL, host, change)
509
510
511def GetChange(host, change):
512 """Query a gerrit server for information about a single change."""
513 path = 'changes/%s' % change
514 return ReadHttpJsonResponse(CreateHttpConn(host, path))
515
516
Andrii Shyshkalovc6c8b4c2016-11-09 20:51:20 +0100517def GetChangeDetail(host, change, o_params=None, ignore_404=True):
szager@chromium.orgb4696232013-10-16 19:45:35 +0000518 """Query a gerrit server for extended information about a single change."""
Andrii Shyshkalovc6c8b4c2016-11-09 20:51:20 +0100519 # TODO(tandrii): cahnge ignore_404 to False by default.
szager@chromium.orgb4696232013-10-16 19:45:35 +0000520 path = 'changes/%s/detail' % change
521 if o_params:
522 path += '?%s' % '&'.join(['o=%s' % p for p in o_params])
Andrii Shyshkalovc6c8b4c2016-11-09 20:51:20 +0100523 return ReadHttpJsonResponse(CreateHttpConn(host, path), ignore_404=ignore_404)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000524
525
agable32978d92016-11-01 12:55:02 -0700526def GetChangeCommit(host, change, revision='current'):
527 """Query a gerrit server for a revision associated with a change."""
528 path = 'changes/%s/revisions/%s/commit?links' % (change, revision)
529 return ReadHttpJsonResponse(CreateHttpConn(host, path))
530
531
szager@chromium.orgb4696232013-10-16 19:45:35 +0000532def GetChangeCurrentRevision(host, change):
533 """Get information about the latest revision for a given change."""
534 return QueryChanges(host, {}, change, o_params=('CURRENT_REVISION',))
535
536
537def GetChangeRevisions(host, change):
538 """Get information about all revisions associated with a change."""
539 return QueryChanges(host, {}, change, o_params=('ALL_REVISIONS',))
540
541
542def GetChangeReview(host, change, revision=None):
543 """Get the current review information for a change."""
544 if not revision:
545 jmsg = GetChangeRevisions(host, change)
546 if not jmsg:
547 return None
548 elif len(jmsg) > 1:
549 raise GerritError(200, 'Multiple changes found for ChangeId %s.' % change)
550 revision = jmsg[0]['current_revision']
551 path = 'changes/%s/revisions/%s/review'
552 return ReadHttpJsonResponse(CreateHttpConn(host, path))
553
554
555def AbandonChange(host, change, msg=''):
556 """Abandon a gerrit change."""
557 path = 'changes/%s/abandon' % change
tandrii@chromium.orgc7da66a2016-03-24 09:52:24 +0000558 body = {'message': msg} if msg else {}
szager@chromium.orgb4696232013-10-16 19:45:35 +0000559 conn = CreateHttpConn(host, path, reqtype='POST', body=body)
560 return ReadHttpJsonResponse(conn, ignore_404=False)
561
562
563def RestoreChange(host, change, msg=''):
564 """Restore a previously abandoned change."""
565 path = 'changes/%s/restore' % change
tandrii@chromium.orgc7da66a2016-03-24 09:52:24 +0000566 body = {'message': msg} if msg else {}
szager@chromium.orgb4696232013-10-16 19:45:35 +0000567 conn = CreateHttpConn(host, path, reqtype='POST', body=body)
568 return ReadHttpJsonResponse(conn, ignore_404=False)
569
570
571def SubmitChange(host, change, wait_for_merge=True):
572 """Submits a gerrit change via Gerrit."""
573 path = 'changes/%s/submit' % change
574 body = {'wait_for_merge': wait_for_merge}
575 conn = CreateHttpConn(host, path, reqtype='POST', body=body)
576 return ReadHttpJsonResponse(conn, ignore_404=False)
577
578
dsansomee2d6fd92016-09-08 00:10:47 -0700579def HasPendingChangeEdit(host, change):
580 conn = CreateHttpConn(host, 'changes/%s/edit' % change)
581 try:
582 ReadHttpResponse(conn, ignore_404=False)
583 except GerritError as e:
584 # On success, gerrit returns status 204; anything else is an error.
585 if e.http_status != 204:
586 raise
587 return False
588 else:
589 return True
590
591
592def DeletePendingChangeEdit(host, change):
593 conn = CreateHttpConn(host, 'changes/%s/edit' % change, reqtype='DELETE')
594 try:
595 ReadHttpResponse(conn, ignore_404=False)
596 except GerritError as e:
597 # On success, gerrit returns status 204; if the edit was already deleted it
598 # returns 404. Anything else is an error.
599 if e.http_status not in (204, 404):
600 raise
601
602
Andrii Shyshkalovea4fc832016-12-01 14:53:23 +0100603def SetCommitMessage(host, change, description, notify='ALL'):
scottmg@chromium.org6d1266e2016-04-26 11:12:26 +0000604 """Updates a commit message."""
scottmg@chromium.org6d1266e2016-04-26 11:12:26 +0000605 try:
Aaron Gablee9373d62016-12-13 12:28:45 -0800606 assert notify in ('ALL', 'NONE')
607 # First, edit the commit message in a draft.
608 path = 'changes/%s/edit:message' % change
609 body = {'message': description}
610 conn = CreateHttpConn(host, path, reqtype='PUT', body=body)
611 try:
612 ReadHttpResponse(conn, ignore_404=False)
613 except GerritError as e:
614 # On success, gerrit returns status 204; anything else is an error.
615 if e.http_status != 204:
616 raise
617 else:
618 raise GerritError(
619 'Unexpectedly received a 200 http status while editing message in '
620 'change %s' % change)
scottmg@chromium.org6d1266e2016-04-26 11:12:26 +0000621
Aaron Gablee9373d62016-12-13 12:28:45 -0800622 # And then publish it.
623 path = 'changes/%s/edit:publish' % change
624 conn = CreateHttpConn(host, path, reqtype='POST', body={'notify': notify})
625 try:
626 ReadHttpResponse(conn, ignore_404=False)
627 except GerritError as e:
628 # On success, gerrit returns status 204; anything else is an error.
629 if e.http_status != 204:
630 raise
631 else:
632 raise GerritError(
633 'Unexpectedly received a 200 http status while publishing message '
634 'change in %s' % change)
635 except (GerritError, KeyboardInterrupt) as e:
636 # Something went wrong with one of the two calls, so we want to clean up
637 # after ourselves before returning.
638 try:
639 DeletePendingChangeEdit(host, change)
640 except GerritError:
641 LOGGER.error('Encountered error while cleaning up after failed attempt '
642 'to set the CL description. You may have to delete the '
643 'pending change edit yourself in the web UI.')
644 raise e
scottmg@chromium.org6d1266e2016-04-26 11:12:26 +0000645
646
szager@chromium.orgb4696232013-10-16 19:45:35 +0000647def GetReviewers(host, change):
648 """Get information about all reviewers attached to a change."""
649 path = 'changes/%s/reviewers' % change
650 return ReadHttpJsonResponse(CreateHttpConn(host, path))
651
652
653def GetReview(host, change, revision):
654 """Get review information about a specific revision of a change."""
655 path = 'changes/%s/revisions/%s/review' % (change, revision)
656 return ReadHttpJsonResponse(CreateHttpConn(host, path))
657
658
Aaron Gable59f48512017-01-12 10:54:46 -0800659def AddReviewers(host, change, add=None, is_reviewer=True, notify=True):
szager@chromium.orgb4696232013-10-16 19:45:35 +0000660 """Add reviewers to a change."""
Aaron Gabledf86e302016-11-08 10:48:03 -0800661 errors = None
szager@chromium.orgb4696232013-10-16 19:45:35 +0000662 if not add:
Aaron Gabledf86e302016-11-08 10:48:03 -0800663 return None
szager@chromium.orgb4696232013-10-16 19:45:35 +0000664 if isinstance(add, basestring):
665 add = (add,)
666 path = 'changes/%s/reviewers' % change
667 for r in add:
Aaron Gabledf86e302016-11-08 10:48:03 -0800668 state = 'REVIEWER' if is_reviewer else 'CC'
Aaron Gable59f48512017-01-12 10:54:46 -0800669 notify = 'ALL' if notify else 'NONE'
tandrii88189772016-09-29 04:29:57 -0700670 body = {
671 'reviewer': r,
Aaron Gabledf86e302016-11-08 10:48:03 -0800672 'state': state,
Aaron Gable59f48512017-01-12 10:54:46 -0800673 'notify': notify,
tandrii88189772016-09-29 04:29:57 -0700674 }
Aaron Gabledf86e302016-11-08 10:48:03 -0800675 try:
676 conn = CreateHttpConn(host, path, reqtype='POST', body=body)
677 _ = ReadHttpJsonResponse(conn, ignore_404=False)
678 except GerritError as e:
679 if e.http_status == 422: # "Unprocessable Entity"
Aaron Gableb7cb65a2017-03-14 11:39:41 -0700680 LOGGER.warn('Note: "%s" not added as a %s' % (r, state.lower()))
Aaron Gabledf86e302016-11-08 10:48:03 -0800681 errors = True
682 else:
683 raise
684 return errors
szager@chromium.orgb4696232013-10-16 19:45:35 +0000685
686
687def RemoveReviewers(host, change, remove=None):
688 """Remove reveiewers from a change."""
689 if not remove:
690 return
691 if isinstance(remove, basestring):
692 remove = (remove,)
693 for r in remove:
694 path = 'changes/%s/reviewers/%s' % (change, r)
695 conn = CreateHttpConn(host, path, reqtype='DELETE')
696 try:
697 ReadHttpResponse(conn, ignore_404=False)
698 except GerritError as e:
699 # On success, gerrit returns status 204; anything else is an error.
700 if e.http_status != 204:
701 raise
702 else:
703 raise GerritError(
704 'Unexpectedly received a 200 http status while deleting reviewer "%s"'
705 ' from change %s' % (r, change))
706
707
708def SetReview(host, change, msg=None, labels=None, notify=None):
709 """Set labels and/or add a message to a code review."""
710 if not msg and not labels:
711 return
712 path = 'changes/%s/revisions/current/review' % change
713 body = {}
714 if msg:
715 body['message'] = msg
716 if labels:
717 body['labels'] = labels
718 if notify:
719 body['notify'] = notify
720 conn = CreateHttpConn(host, path, reqtype='POST', body=body)
721 response = ReadHttpJsonResponse(conn)
722 if labels:
723 for key, val in labels.iteritems():
724 if ('labels' not in response or key not in response['labels'] or
725 int(response['labels'][key] != int(val))):
726 raise GerritError(200, 'Unable to set "%s" label on change %s.' % (
727 key, change))
728
729
730def ResetReviewLabels(host, change, label, value='0', message=None,
731 notify=None):
732 """Reset the value of a given label for all reviewers on a change."""
733 # This is tricky, because we want to work on the "current revision", but
734 # there's always the risk that "current revision" will change in between
735 # API calls. So, we check "current revision" at the beginning and end; if
736 # it has changed, raise an exception.
737 jmsg = GetChangeCurrentRevision(host, change)
738 if not jmsg:
739 raise GerritError(
740 200, 'Could not get review information for change "%s"' % change)
741 value = str(value)
742 revision = jmsg[0]['current_revision']
743 path = 'changes/%s/revisions/%s/review' % (change, revision)
744 message = message or (
745 '%s label set to %s programmatically.' % (label, value))
746 jmsg = GetReview(host, change, revision)
747 if not jmsg:
748 raise GerritError(200, 'Could not get review information for revison %s '
749 'of change %s' % (revision, change))
750 for review in jmsg.get('labels', {}).get(label, {}).get('all', []):
751 if str(review.get('value', value)) != value:
752 body = {
753 'message': message,
754 'labels': {label: value},
755 'on_behalf_of': review['_account_id'],
756 }
757 if notify:
758 body['notify'] = notify
759 conn = CreateHttpConn(
760 host, path, reqtype='POST', body=body)
761 response = ReadHttpJsonResponse(conn)
762 if str(response['labels'][label]) != value:
763 username = review.get('email', jmsg.get('name', ''))
764 raise GerritError(200, 'Unable to set %s label for user "%s"'
765 ' on change %s.' % (label, username, change))
766 jmsg = GetChangeCurrentRevision(host, change)
767 if not jmsg:
768 raise GerritError(
769 200, 'Could not get review information for change "%s"' % change)
770 elif jmsg[0]['current_revision'] != revision:
771 raise GerritError(200, 'While resetting labels on change "%s", '
772 'a new patchset was uploaded.' % change)
Dan Jacques8d11e482016-11-15 14:25:56 -0800773
774
dimu833c94c2017-01-18 17:36:15 -0800775def CreateGerritBranch(host, project, branch, commit):
776 """
777 Create a new branch from given project and commit
778 https://gerrit-review.googlesource.com/Documentation/rest-api-projects.html#create-branch
779
780 Returns:
781 A JSON with 'ref' key
782 """
783 path = 'projects/%s/branches/%s' % (project, branch)
784 body = {'revision': commit}
785 conn = CreateHttpConn(host, path, reqtype='PUT', body=body)
786 response = ReadHttpJsonResponse(conn)
787 if response:
788 return response
789 raise GerritError(200, 'Unable to create gerrit branch')
790
791
792def GetGerritBranch(host, project, branch):
793 """
794 Get a branch from given project and commit
795 https://gerrit-review.googlesource.com/Documentation/rest-api-projects.html#get-branch
796
797 Returns:
798 A JSON object with 'revision' key
799 """
800 path = 'projects/%s/branches/%s' % (project, branch)
801 conn = CreateHttpConn(host, path, reqtype='GET')
802 response = ReadHttpJsonResponse(conn)
803 if response:
804 return response
805 raise GerritError(200, 'Unable to get gerrit branch')
806
807
Dan Jacques8d11e482016-11-15 14:25:56 -0800808@contextlib.contextmanager
809def tempdir():
810 tdir = None
811 try:
812 tdir = tempfile.mkdtemp(suffix='gerrit_util')
813 yield tdir
814 finally:
815 if tdir:
816 gclient_utils.rmtree(tdir)