blob: 2f274ce3d7eddfd0b1b4b3182ba7b61c7c34948a [file] [log] [blame]
szager@chromium.orgb4696232013-10-16 19:45:35 +00001# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""
6Utilities for requesting information for a gerrit server via https.
7
8https://gerrit-review.googlesource.com/Documentation/rest-api.html
9"""
10
11import base64
Dan Jacques8d11e482016-11-15 14:25:56 -080012import contextlib
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +000013import cookielib
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +010014import httplib # Still used for its constants.
szager@chromium.orgb4696232013-10-16 19:45:35 +000015import json
16import logging
17import netrc
18import os
nodir@chromium.orgce32b6e2014-05-12 20:31:32 +000019import re
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +000020import socket
szager@chromium.orgf202a252014-05-27 18:55:52 +000021import stat
22import sys
Dan Jacques8d11e482016-11-15 14:25:56 -080023import tempfile
szager@chromium.orgb4696232013-10-16 19:45:35 +000024import time
25import urllib
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +000026import urlparse
szager@chromium.orgb4696232013-10-16 19:45:35 +000027from cStringIO import StringIO
28
Dan Jacques8d11e482016-11-15 14:25:56 -080029import gclient_utils
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +010030from third_party import httplib2
szager@chromium.orgf202a252014-05-27 18:55:52 +000031
szager@chromium.orgb4696232013-10-16 19:45:35 +000032LOGGER = logging.getLogger()
33TRY_LIMIT = 5
34
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +000035
szager@chromium.orgb4696232013-10-16 19:45:35 +000036# Controls the transport protocol used to communicate with gerrit.
37# This is parameterized primarily to enable GerritTestCase.
38GERRIT_PROTOCOL = 'https'
39
40
41class GerritError(Exception):
42 """Exception class for errors commuicating with the gerrit-on-borg service."""
43 def __init__(self, http_status, *args, **kwargs):
44 super(GerritError, self).__init__(*args, **kwargs)
45 self.http_status = http_status
46 self.message = '(%d) %s' % (self.http_status, self.message)
47
48
nodir@chromium.orgce32b6e2014-05-12 20:31:32 +000049class GerritAuthenticationError(GerritError):
50 """Exception class for authentication errors during Gerrit communication."""
51
52
szager@chromium.orgb4696232013-10-16 19:45:35 +000053def _QueryString(param_dict, first_param=None):
54 """Encodes query parameters in the key:val[+key:val...] format specified here:
55
56 https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#list-changes
57 """
58 q = [urllib.quote(first_param)] if first_param else []
59 q.extend(['%s:%s' % (key, val) for key, val in param_dict.iteritems()])
60 return '+'.join(q)
61
62
63def GetConnectionClass(protocol=None):
64 if protocol is None:
65 protocol = GERRIT_PROTOCOL
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +010066 if protocol in ('http', 'https'):
67 return httplib2.Http
szager@chromium.orgb4696232013-10-16 19:45:35 +000068 else:
69 raise RuntimeError(
70 "Don't know how to work with protocol '%s'" % protocol)
71
72
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +000073class Authenticator(object):
74 """Base authenticator class for authenticator implementations to subclass."""
75
76 def get_auth_header(self, host):
77 raise NotImplementedError()
78
79 @staticmethod
80 def get():
81 """Returns: (Authenticator) The identified Authenticator to use.
82
83 Probes the local system and its environment and identifies the
84 Authenticator instance to use.
85 """
86 if GceAuthenticator.is_gce():
87 return GceAuthenticator()
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +000088 return CookiesAuthenticator()
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +000089
90
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +000091class CookiesAuthenticator(Authenticator):
92 """Authenticator implementation that uses ".netrc" or ".gitcookies" for token.
93
94 Expected case for developer workstations.
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +000095 """
96
97 def __init__(self):
98 self.netrc = self._get_netrc()
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +000099 self.gitcookies = self._get_gitcookies()
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000100
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000101 @classmethod
102 def get_new_password_message(cls, host):
103 assert not host.startswith('http')
104 # Assume *.googlesource.com pattern.
105 parts = host.split('.')
106 if not parts[0].endswith('-review'):
107 parts[0] += '-review'
108 url = 'https://%s/new-password' % ('.'.join(parts))
Andrii Shyshkalov0a0b0672017-03-16 16:27:48 +0100109 return 'You can (re)generate your credentials by visiting %s' % url
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000110
111 @classmethod
112 def get_netrc_path(cls):
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000113 path = '_netrc' if sys.platform.startswith('win') else '.netrc'
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000114 return os.path.expanduser(os.path.join('~', path))
115
116 @classmethod
117 def _get_netrc(cls):
Dan Jacques8d11e482016-11-15 14:25:56 -0800118 # Buffer the '.netrc' path. Use an empty file if it doesn't exist.
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000119 path = cls.get_netrc_path()
Dan Jacques8d11e482016-11-15 14:25:56 -0800120 content = ''
121 if os.path.exists(path):
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000122 st = os.stat(path)
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000123 if st.st_mode & (stat.S_IRWXG | stat.S_IRWXO):
124 print >> sys.stderr, (
125 'WARNING: netrc file %s cannot be used because its file '
126 'permissions are insecure. netrc file permissions should be '
127 '600.' % path)
Dan Jacques8d11e482016-11-15 14:25:56 -0800128 with open(path) as fd:
129 content = fd.read()
130
131 # Load the '.netrc' file. We strip comments from it because processing them
132 # can trigger a bug in Windows. See crbug.com/664664.
133 content = '\n'.join(l for l in content.splitlines()
134 if l.strip() and not l.strip().startswith('#'))
135 with tempdir() as tdir:
136 netrc_path = os.path.join(tdir, 'netrc')
137 with open(netrc_path, 'w') as fd:
138 fd.write(content)
139 os.chmod(netrc_path, (stat.S_IRUSR | stat.S_IWUSR))
140 return cls._get_netrc_from_path(netrc_path)
141
142 @classmethod
143 def _get_netrc_from_path(cls, path):
144 try:
145 return netrc.netrc(path)
146 except IOError:
147 print >> sys.stderr, 'WARNING: Could not read netrc file %s' % path
148 return netrc.netrc(os.devnull)
149 except netrc.NetrcParseError as e:
150 print >> sys.stderr, ('ERROR: Cannot use netrc file %s due to a '
151 'parsing error: %s' % (path, e))
152 return netrc.netrc(os.devnull)
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000153
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000154 @classmethod
155 def get_gitcookies_path(cls):
Ravi Mistry0bfa9ad2016-11-21 12:58:31 -0500156 if os.getenv('GIT_COOKIES_PATH'):
157 return os.getenv('GIT_COOKIES_PATH')
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000158 return os.path.join(os.environ['HOME'], '.gitcookies')
159
160 @classmethod
161 def _get_gitcookies(cls):
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +0000162 gitcookies = {}
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000163 path = cls.get_gitcookies_path()
164 if not os.path.exists(path):
165 return gitcookies
166
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +0000167 try:
168 f = open(path, 'rb')
169 except IOError:
170 return gitcookies
171
172 with f:
173 for line in f:
174 try:
175 fields = line.strip().split('\t')
176 if line.strip().startswith('#') or len(fields) != 7:
177 continue
178 domain, xpath, key, value = fields[0], fields[2], fields[5], fields[6]
179 if xpath == '/' and key == 'o':
180 login, secret_token = value.split('=', 1)
181 gitcookies[domain] = (login, secret_token)
182 except (IndexError, ValueError, TypeError) as exc:
Andrii Shyshkalov5b04a572017-01-23 17:44:41 +0100183 LOGGER.warning(exc)
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +0000184
185 return gitcookies
186
Andrii Shyshkalovbb86fbb2017-03-24 14:59:28 +0100187 def _get_auth_for_host(self, host):
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +0000188 for domain, creds in self.gitcookies.iteritems():
189 if cookielib.domain_match(host, domain):
Andrii Shyshkalovbb86fbb2017-03-24 14:59:28 +0100190 return (creds[0], None, creds[1])
191 return self.netrc.authenticators(host)
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +0000192
Andrii Shyshkalovbb86fbb2017-03-24 14:59:28 +0100193 def get_auth_header(self, host):
194 auth = self._get_auth_for_host(host)
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000195 if auth:
196 return 'Basic %s' % (base64.b64encode('%s:%s' % (auth[0], auth[2])))
197 return None
198
Andrii Shyshkalovbb86fbb2017-03-24 14:59:28 +0100199 def get_auth_email(self, host):
200 """Best effort parsing of email to be used for auth for the given host."""
201 auth = self._get_auth_for_host(host)
202 if not auth:
203 return None
204 login = auth[0]
205 # login typically looks like 'git-xxx.example.com'
206 if not login.startswith('git-') or '.' not in login:
207 return None
208 username, domain = login[len('git-'):].split('.', 1)
209 return '%s@%s' % (username, domain)
210
Andrii Shyshkalov18975322017-01-25 16:44:13 +0100211
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000212# Backwards compatibility just in case somebody imports this outside of
213# depot_tools.
214NetrcAuthenticator = CookiesAuthenticator
215
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000216
217class GceAuthenticator(Authenticator):
218 """Authenticator implementation that uses GCE metadata service for token.
219 """
220
221 _INFO_URL = 'http://metadata.google.internal'
222 _ACQUIRE_URL = ('http://metadata/computeMetadata/v1/instance/'
223 'service-accounts/default/token')
224 _ACQUIRE_HEADERS = {"Metadata-Flavor": "Google"}
225
226 _cache_is_gce = None
227 _token_cache = None
228 _token_expiration = None
229
230 @classmethod
231 def is_gce(cls):
Ravi Mistryfad941b2016-11-15 13:00:47 -0500232 if os.getenv('SKIP_GCE_AUTH_FOR_GIT'):
233 return False
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000234 if cls._cache_is_gce is None:
235 cls._cache_is_gce = cls._test_is_gce()
236 return cls._cache_is_gce
237
238 @classmethod
239 def _test_is_gce(cls):
240 # Based on https://cloud.google.com/compute/docs/metadata#runninggce
241 try:
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100242 resp, _ = cls._get(cls._INFO_URL)
243 except (socket.error, httplib2.ServerNotFoundError):
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000244 # Could not resolve URL.
245 return False
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100246 return resp.get('metadata-flavor') == 'Google'
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000247
248 @staticmethod
249 def _get(url, **kwargs):
250 next_delay_sec = 1
251 for i in xrange(TRY_LIMIT):
252 if i > 0:
253 # Retry server error status codes.
254 LOGGER.info('Encountered server error; retrying after %d second(s).',
255 next_delay_sec)
256 time.sleep(next_delay_sec)
257 next_delay_sec *= 2
258
259 p = urlparse.urlparse(url)
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100260 c = GetConnectionClass(protocol=p.scheme)()
261 resp, contents = c.request(url, 'GET', **kwargs)
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000262 LOGGER.debug('GET [%s] #%d/%d (%d)', url, i+1, TRY_LIMIT, resp.status)
263 if resp.status < httplib.INTERNAL_SERVER_ERROR:
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100264 return (resp, contents)
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000265
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000266 @classmethod
267 def _get_token_dict(cls):
268 if cls._token_cache:
269 # If it expires within 25 seconds, refresh.
270 if cls._token_expiration < time.time() - 25:
271 return cls._token_cache
272
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100273 resp, contents = cls._get(cls._ACQUIRE_URL, headers=cls._ACQUIRE_HEADERS)
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000274 if resp.status != httplib.OK:
275 return None
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100276 cls._token_cache = json.loads(contents)
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000277 cls._token_expiration = cls._token_cache['expires_in'] + time.time()
278 return cls._token_cache
279
280 def get_auth_header(self, _host):
281 token_dict = self._get_token_dict()
282 if not token_dict:
283 return None
284 return '%(token_type)s %(access_token)s' % token_dict
285
286
szager@chromium.orgb4696232013-10-16 19:45:35 +0000287def CreateHttpConn(host, path, reqtype='GET', headers=None, body=None):
288 """Opens an https connection to a gerrit service, and sends a request."""
289 headers = headers or {}
290 bare_host = host.partition(':')[0]
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000291
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000292 auth = Authenticator.get().get_auth_header(bare_host)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000293 if auth:
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000294 headers.setdefault('Authorization', auth)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000295 else:
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000296 LOGGER.debug('No authorization found for %s.' % bare_host)
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000297
Dan Jacques6d5bcc22016-11-14 15:32:04 -0800298 url = path
299 if not url.startswith('/'):
300 url = '/' + url
301 if 'Authorization' in headers and not url.startswith('/a/'):
302 url = '/a%s' % url
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000303
szager@chromium.orgb4696232013-10-16 19:45:35 +0000304 if body:
305 body = json.JSONEncoder().encode(body)
306 headers.setdefault('Content-Type', 'application/json')
307 if LOGGER.isEnabledFor(logging.DEBUG):
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000308 LOGGER.debug('%s %s://%s%s' % (reqtype, GERRIT_PROTOCOL, host, url))
szager@chromium.orgb4696232013-10-16 19:45:35 +0000309 for key, val in headers.iteritems():
310 if key == 'Authorization':
311 val = 'HIDDEN'
312 LOGGER.debug('%s: %s' % (key, val))
313 if body:
314 LOGGER.debug(body)
315 conn = GetConnectionClass()(host)
316 conn.req_host = host
317 conn.req_params = {
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100318 'uri': urlparse.urljoin('%s://%s' % (GERRIT_PROTOCOL, host), url),
szager@chromium.orgb4696232013-10-16 19:45:35 +0000319 'method': reqtype,
320 'headers': headers,
321 'body': body,
322 }
szager@chromium.orgb4696232013-10-16 19:45:35 +0000323 return conn
324
325
326def ReadHttpResponse(conn, expect_status=200, ignore_404=True):
327 """Reads an http response from a connection into a string buffer.
328
329 Args:
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100330 conn: An Http object created by CreateHttpConn above.
szager@chromium.orgb4696232013-10-16 19:45:35 +0000331 expect_status: Success is indicated by this status in the response.
332 ignore_404: For many requests, gerrit-on-borg will return 404 if the request
333 doesn't match the database contents. In most such cases, we
334 want the API to return None rather than raise an Exception.
335 Returns: A string buffer containing the connection's reply.
336 """
337
338 sleep_time = 0.5
339 for idx in range(TRY_LIMIT):
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100340 response, contents = conn.request(**conn.req_params)
nodir@chromium.orgce32b6e2014-05-12 20:31:32 +0000341
342 # Check if this is an authentication issue.
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100343 www_authenticate = response.get('www-authenticate')
nodir@chromium.orgce32b6e2014-05-12 20:31:32 +0000344 if (response.status in (httplib.UNAUTHORIZED, httplib.FOUND) and
345 www_authenticate):
346 auth_match = re.search('realm="([^"]+)"', www_authenticate, re.I)
347 host = auth_match.group(1) if auth_match else conn.req_host
348 reason = ('Authentication failed. Please make sure your .netrc file '
349 'has credentials for %s' % host)
350 raise GerritAuthenticationError(response.status, reason)
351
szager@chromium.orgb4696232013-10-16 19:45:35 +0000352 # If response.status < 500 then the result is final; break retry loop.
353 if response.status < 500:
Andrii Shyshkalov5b04a572017-01-23 17:44:41 +0100354 LOGGER.debug('got response %d for %s %s', response.status,
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100355 conn.req_params['method'], conn.req_params['uri'])
szager@chromium.orgb4696232013-10-16 19:45:35 +0000356 break
357 # A status >=500 is assumed to be a possible transient error; retry.
358 http_version = 'HTTP/%s' % ('1.1' if response.version == 11 else '1.0')
Andrii Shyshkalov5b04a572017-01-23 17:44:41 +0100359 LOGGER.warn('A transient error occurred while querying %s:\n'
360 '%s %s %s\n'
361 '%s %d %s',
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100362 conn.host, conn.req_params['method'], conn.req_params['uri'],
Andrii Shyshkalov5b04a572017-01-23 17:44:41 +0100363 http_version, http_version, response.status, response.reason)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000364 if TRY_LIMIT - idx > 1:
Andrii Shyshkalov5b04a572017-01-23 17:44:41 +0100365 LOGGER.warn('... will retry %d more times.', TRY_LIMIT - idx - 1)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000366 time.sleep(sleep_time)
367 sleep_time = sleep_time * 2
szager@chromium.orgb4696232013-10-16 19:45:35 +0000368 if ignore_404 and response.status == 404:
369 return StringIO()
370 if response.status != expect_status:
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100371 reason = '%s: %s' % (response.reason, contents)
nodir@chromium.orga7798032014-04-30 23:40:53 +0000372 raise GerritError(response.status, reason)
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100373 return StringIO(contents)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000374
375
376def ReadHttpJsonResponse(conn, expect_status=200, ignore_404=True):
377 """Parses an https response as json."""
378 fh = ReadHttpResponse(
379 conn, expect_status=expect_status, ignore_404=ignore_404)
380 # The first line of the response should always be: )]}'
381 s = fh.readline()
382 if s and s.rstrip() != ")]}'":
383 raise GerritError(200, 'Unexpected json output: %s' % s)
384 s = fh.read()
385 if not s:
386 return None
387 return json.loads(s)
388
389
390def QueryChanges(host, param_dict, first_param=None, limit=None, o_params=None,
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100391 start=None):
szager@chromium.orgb4696232013-10-16 19:45:35 +0000392 """
393 Queries a gerrit-on-borg server for changes matching query terms.
394
395 Args:
396 param_dict: A dictionary of search parameters, as documented here:
397 http://gerrit-documentation.googlecode.com/svn/Documentation/2.6/user-search.html
398 first_param: A change identifier
399 limit: Maximum number of results to return.
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100400 start: how many changes to skip (starting with the most recent)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000401 o_params: A list of additional output specifiers, as documented here:
402 https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#list-changes
403 Returns:
404 A list of json-decoded query results.
405 """
406 # Note that no attempt is made to escape special characters; YMMV.
407 if not param_dict and not first_param:
408 raise RuntimeError('QueryChanges requires search parameters')
409 path = 'changes/?q=%s' % _QueryString(param_dict, first_param)
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100410 if start:
411 path = '%s&start=%s' % (path, start)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000412 if limit:
413 path = '%s&n=%d' % (path, limit)
414 if o_params:
415 path = '%s&%s' % (path, '&'.join(['o=%s' % p for p in o_params]))
416 # Don't ignore 404; a query should always return a list, even if it's empty.
417 return ReadHttpJsonResponse(CreateHttpConn(host, path), ignore_404=False)
418
419
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000420def GenerateAllChanges(host, param_dict, first_param=None, limit=500,
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100421 o_params=None, start=None):
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000422 """
423 Queries a gerrit-on-borg server for all the changes matching the query terms.
424
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100425 WARNING: this is unreliable if a change matching the query is modified while
426 this function is being called.
427
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000428 A single query to gerrit-on-borg is limited on the number of results by the
429 limit parameter on the request (see QueryChanges) and the server maximum
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100430 limit.
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000431
432 Args:
433 param_dict, first_param: Refer to QueryChanges().
434 limit: Maximum number of requested changes per query.
435 o_params: Refer to QueryChanges().
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100436 start: Refer to QueryChanges().
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000437
438 Returns:
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100439 A generator object to the list of returned changes.
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000440 """
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100441 already_returned = set()
442 def at_most_once(cls):
443 for cl in cls:
444 if cl['_number'] not in already_returned:
445 already_returned.add(cl['_number'])
446 yield cl
447
448 start = start or 0
449 cur_start = start
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000450 more_changes = True
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100451
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000452 while more_changes:
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100453 # This will fetch changes[start..start+limit] sorted by most recently
454 # updated. Since the rank of any change in this list can be changed any time
455 # (say user posting comment), subsequent calls may overalp like this:
456 # > initial order ABCDEFGH
457 # query[0..3] => ABC
458 # > E get's updated. New order: EABCDFGH
459 # query[3..6] => CDF # C is a dup
460 # query[6..9] => GH # E is missed.
461 page = QueryChanges(host, param_dict, first_param, limit, o_params,
462 cur_start)
463 for cl in at_most_once(page):
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000464 yield cl
465
466 more_changes = [cl for cl in page if '_more_changes' in cl]
467 if len(more_changes) > 1:
468 raise GerritError(
469 200,
470 'Received %d changes with a _more_changes attribute set but should '
471 'receive at most one.' % len(more_changes))
472 if more_changes:
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100473 cur_start += len(page)
474
475 # If we paged through, query again the first page which in most circumstances
476 # will fetch all changes that were modified while this function was run.
477 if start != cur_start:
478 page = QueryChanges(host, param_dict, first_param, limit, o_params, start)
479 for cl in at_most_once(page):
480 yield cl
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000481
482
szager@chromium.orgb4696232013-10-16 19:45:35 +0000483def MultiQueryChanges(host, param_dict, change_list, limit=None, o_params=None,
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100484 start=None):
szager@chromium.orgb4696232013-10-16 19:45:35 +0000485 """Initiate a query composed of multiple sets of query parameters."""
486 if not change_list:
487 raise RuntimeError(
488 "MultiQueryChanges requires a list of change numbers/id's")
489 q = ['q=%s' % '+OR+'.join([urllib.quote(str(x)) for x in change_list])]
490 if param_dict:
491 q.append(_QueryString(param_dict))
492 if limit:
493 q.append('n=%d' % limit)
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100494 if start:
495 q.append('S=%s' % start)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000496 if o_params:
497 q.extend(['o=%s' % p for p in o_params])
498 path = 'changes/?%s' % '&'.join(q)
499 try:
500 result = ReadHttpJsonResponse(CreateHttpConn(host, path), ignore_404=False)
501 except GerritError as e:
502 msg = '%s:\n%s' % (e.message, path)
503 raise GerritError(e.http_status, msg)
504 return result
505
506
507def GetGerritFetchUrl(host):
508 """Given a gerrit host name returns URL of a gerrit instance to fetch from."""
509 return '%s://%s/' % (GERRIT_PROTOCOL, host)
510
511
512def GetChangePageUrl(host, change_number):
513 """Given a gerrit host name and change number, return change page url."""
514 return '%s://%s/#/c/%d/' % (GERRIT_PROTOCOL, host, change_number)
515
516
517def GetChangeUrl(host, change):
518 """Given a gerrit host name and change id, return an url for the change."""
519 return '%s://%s/a/changes/%s' % (GERRIT_PROTOCOL, host, change)
520
521
522def GetChange(host, change):
523 """Query a gerrit server for information about a single change."""
524 path = 'changes/%s' % change
525 return ReadHttpJsonResponse(CreateHttpConn(host, path))
526
527
Andrii Shyshkalovc6c8b4c2016-11-09 20:51:20 +0100528def GetChangeDetail(host, change, o_params=None, ignore_404=True):
szager@chromium.orgb4696232013-10-16 19:45:35 +0000529 """Query a gerrit server for extended information about a single change."""
Andrii Shyshkalovc6c8b4c2016-11-09 20:51:20 +0100530 # TODO(tandrii): cahnge ignore_404 to False by default.
szager@chromium.orgb4696232013-10-16 19:45:35 +0000531 path = 'changes/%s/detail' % change
532 if o_params:
533 path += '?%s' % '&'.join(['o=%s' % p for p in o_params])
Andrii Shyshkalovc6c8b4c2016-11-09 20:51:20 +0100534 return ReadHttpJsonResponse(CreateHttpConn(host, path), ignore_404=ignore_404)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000535
536
agable32978d92016-11-01 12:55:02 -0700537def GetChangeCommit(host, change, revision='current'):
538 """Query a gerrit server for a revision associated with a change."""
539 path = 'changes/%s/revisions/%s/commit?links' % (change, revision)
540 return ReadHttpJsonResponse(CreateHttpConn(host, path))
541
542
szager@chromium.orgb4696232013-10-16 19:45:35 +0000543def GetChangeCurrentRevision(host, change):
544 """Get information about the latest revision for a given change."""
545 return QueryChanges(host, {}, change, o_params=('CURRENT_REVISION',))
546
547
548def GetChangeRevisions(host, change):
549 """Get information about all revisions associated with a change."""
550 return QueryChanges(host, {}, change, o_params=('ALL_REVISIONS',))
551
552
553def GetChangeReview(host, change, revision=None):
554 """Get the current review information for a change."""
555 if not revision:
556 jmsg = GetChangeRevisions(host, change)
557 if not jmsg:
558 return None
559 elif len(jmsg) > 1:
560 raise GerritError(200, 'Multiple changes found for ChangeId %s.' % change)
561 revision = jmsg[0]['current_revision']
562 path = 'changes/%s/revisions/%s/review'
563 return ReadHttpJsonResponse(CreateHttpConn(host, path))
564
565
566def AbandonChange(host, change, msg=''):
567 """Abandon a gerrit change."""
568 path = 'changes/%s/abandon' % change
tandrii@chromium.orgc7da66a2016-03-24 09:52:24 +0000569 body = {'message': msg} if msg else {}
szager@chromium.orgb4696232013-10-16 19:45:35 +0000570 conn = CreateHttpConn(host, path, reqtype='POST', body=body)
571 return ReadHttpJsonResponse(conn, ignore_404=False)
572
573
574def RestoreChange(host, change, msg=''):
575 """Restore a previously abandoned change."""
576 path = 'changes/%s/restore' % change
tandrii@chromium.orgc7da66a2016-03-24 09:52:24 +0000577 body = {'message': msg} if msg else {}
szager@chromium.orgb4696232013-10-16 19:45:35 +0000578 conn = CreateHttpConn(host, path, reqtype='POST', body=body)
579 return ReadHttpJsonResponse(conn, ignore_404=False)
580
581
582def SubmitChange(host, change, wait_for_merge=True):
583 """Submits a gerrit change via Gerrit."""
584 path = 'changes/%s/submit' % change
585 body = {'wait_for_merge': wait_for_merge}
586 conn = CreateHttpConn(host, path, reqtype='POST', body=body)
587 return ReadHttpJsonResponse(conn, ignore_404=False)
588
589
dsansomee2d6fd92016-09-08 00:10:47 -0700590def HasPendingChangeEdit(host, change):
591 conn = CreateHttpConn(host, 'changes/%s/edit' % change)
592 try:
593 ReadHttpResponse(conn, ignore_404=False)
594 except GerritError as e:
595 # On success, gerrit returns status 204; anything else is an error.
596 if e.http_status != 204:
597 raise
598 return False
599 else:
600 return True
601
602
603def DeletePendingChangeEdit(host, change):
604 conn = CreateHttpConn(host, 'changes/%s/edit' % change, reqtype='DELETE')
605 try:
606 ReadHttpResponse(conn, ignore_404=False)
607 except GerritError as e:
608 # On success, gerrit returns status 204; if the edit was already deleted it
609 # returns 404. Anything else is an error.
610 if e.http_status not in (204, 404):
611 raise
612
613
Andrii Shyshkalovea4fc832016-12-01 14:53:23 +0100614def SetCommitMessage(host, change, description, notify='ALL'):
scottmg@chromium.org6d1266e2016-04-26 11:12:26 +0000615 """Updates a commit message."""
scottmg@chromium.org6d1266e2016-04-26 11:12:26 +0000616 try:
Aaron Gablee9373d62016-12-13 12:28:45 -0800617 assert notify in ('ALL', 'NONE')
618 # First, edit the commit message in a draft.
619 path = 'changes/%s/edit:message' % change
620 body = {'message': description}
621 conn = CreateHttpConn(host, path, reqtype='PUT', body=body)
622 try:
623 ReadHttpResponse(conn, ignore_404=False)
624 except GerritError as e:
625 # On success, gerrit returns status 204; anything else is an error.
626 if e.http_status != 204:
627 raise
628 else:
629 raise GerritError(
630 'Unexpectedly received a 200 http status while editing message in '
631 'change %s' % change)
scottmg@chromium.org6d1266e2016-04-26 11:12:26 +0000632
Aaron Gablee9373d62016-12-13 12:28:45 -0800633 # And then publish it.
634 path = 'changes/%s/edit:publish' % change
635 conn = CreateHttpConn(host, path, reqtype='POST', body={'notify': notify})
636 try:
637 ReadHttpResponse(conn, ignore_404=False)
638 except GerritError as e:
639 # On success, gerrit returns status 204; anything else is an error.
640 if e.http_status != 204:
641 raise
642 else:
643 raise GerritError(
644 'Unexpectedly received a 200 http status while publishing message '
645 'change in %s' % change)
646 except (GerritError, KeyboardInterrupt) as e:
647 # Something went wrong with one of the two calls, so we want to clean up
648 # after ourselves before returning.
649 try:
650 DeletePendingChangeEdit(host, change)
651 except GerritError:
652 LOGGER.error('Encountered error while cleaning up after failed attempt '
653 'to set the CL description. You may have to delete the '
654 'pending change edit yourself in the web UI.')
655 raise e
scottmg@chromium.org6d1266e2016-04-26 11:12:26 +0000656
657
szager@chromium.orgb4696232013-10-16 19:45:35 +0000658def GetReviewers(host, change):
659 """Get information about all reviewers attached to a change."""
660 path = 'changes/%s/reviewers' % change
661 return ReadHttpJsonResponse(CreateHttpConn(host, path))
662
663
664def GetReview(host, change, revision):
665 """Get review information about a specific revision of a change."""
666 path = 'changes/%s/revisions/%s/review' % (change, revision)
667 return ReadHttpJsonResponse(CreateHttpConn(host, path))
668
669
Aaron Gable59f48512017-01-12 10:54:46 -0800670def AddReviewers(host, change, add=None, is_reviewer=True, notify=True):
szager@chromium.orgb4696232013-10-16 19:45:35 +0000671 """Add reviewers to a change."""
Aaron Gabledf86e302016-11-08 10:48:03 -0800672 errors = None
szager@chromium.orgb4696232013-10-16 19:45:35 +0000673 if not add:
Aaron Gabledf86e302016-11-08 10:48:03 -0800674 return None
szager@chromium.orgb4696232013-10-16 19:45:35 +0000675 if isinstance(add, basestring):
676 add = (add,)
677 path = 'changes/%s/reviewers' % change
678 for r in add:
Aaron Gabledf86e302016-11-08 10:48:03 -0800679 state = 'REVIEWER' if is_reviewer else 'CC'
Aaron Gable59f48512017-01-12 10:54:46 -0800680 notify = 'ALL' if notify else 'NONE'
tandrii88189772016-09-29 04:29:57 -0700681 body = {
682 'reviewer': r,
Aaron Gabledf86e302016-11-08 10:48:03 -0800683 'state': state,
Aaron Gable59f48512017-01-12 10:54:46 -0800684 'notify': notify,
tandrii88189772016-09-29 04:29:57 -0700685 }
Aaron Gabledf86e302016-11-08 10:48:03 -0800686 try:
687 conn = CreateHttpConn(host, path, reqtype='POST', body=body)
688 _ = ReadHttpJsonResponse(conn, ignore_404=False)
689 except GerritError as e:
690 if e.http_status == 422: # "Unprocessable Entity"
Aaron Gableb7cb65a2017-03-14 11:39:41 -0700691 LOGGER.warn('Note: "%s" not added as a %s' % (r, state.lower()))
Aaron Gabledf86e302016-11-08 10:48:03 -0800692 errors = True
693 else:
694 raise
695 return errors
szager@chromium.orgb4696232013-10-16 19:45:35 +0000696
697
698def RemoveReviewers(host, change, remove=None):
699 """Remove reveiewers from a change."""
700 if not remove:
701 return
702 if isinstance(remove, basestring):
703 remove = (remove,)
704 for r in remove:
705 path = 'changes/%s/reviewers/%s' % (change, r)
706 conn = CreateHttpConn(host, path, reqtype='DELETE')
707 try:
708 ReadHttpResponse(conn, ignore_404=False)
709 except GerritError as e:
710 # On success, gerrit returns status 204; anything else is an error.
711 if e.http_status != 204:
712 raise
713 else:
714 raise GerritError(
715 'Unexpectedly received a 200 http status while deleting reviewer "%s"'
716 ' from change %s' % (r, change))
717
718
719def SetReview(host, change, msg=None, labels=None, notify=None):
720 """Set labels and/or add a message to a code review."""
721 if not msg and not labels:
722 return
723 path = 'changes/%s/revisions/current/review' % change
724 body = {}
725 if msg:
726 body['message'] = msg
727 if labels:
728 body['labels'] = labels
729 if notify:
730 body['notify'] = notify
731 conn = CreateHttpConn(host, path, reqtype='POST', body=body)
732 response = ReadHttpJsonResponse(conn)
733 if labels:
734 for key, val in labels.iteritems():
735 if ('labels' not in response or key not in response['labels'] or
736 int(response['labels'][key] != int(val))):
737 raise GerritError(200, 'Unable to set "%s" label on change %s.' % (
738 key, change))
739
740
741def ResetReviewLabels(host, change, label, value='0', message=None,
742 notify=None):
743 """Reset the value of a given label for all reviewers on a change."""
744 # This is tricky, because we want to work on the "current revision", but
745 # there's always the risk that "current revision" will change in between
746 # API calls. So, we check "current revision" at the beginning and end; if
747 # it has changed, raise an exception.
748 jmsg = GetChangeCurrentRevision(host, change)
749 if not jmsg:
750 raise GerritError(
751 200, 'Could not get review information for change "%s"' % change)
752 value = str(value)
753 revision = jmsg[0]['current_revision']
754 path = 'changes/%s/revisions/%s/review' % (change, revision)
755 message = message or (
756 '%s label set to %s programmatically.' % (label, value))
757 jmsg = GetReview(host, change, revision)
758 if not jmsg:
759 raise GerritError(200, 'Could not get review information for revison %s '
760 'of change %s' % (revision, change))
761 for review in jmsg.get('labels', {}).get(label, {}).get('all', []):
762 if str(review.get('value', value)) != value:
763 body = {
764 'message': message,
765 'labels': {label: value},
766 'on_behalf_of': review['_account_id'],
767 }
768 if notify:
769 body['notify'] = notify
770 conn = CreateHttpConn(
771 host, path, reqtype='POST', body=body)
772 response = ReadHttpJsonResponse(conn)
773 if str(response['labels'][label]) != value:
774 username = review.get('email', jmsg.get('name', ''))
775 raise GerritError(200, 'Unable to set %s label for user "%s"'
776 ' on change %s.' % (label, username, change))
777 jmsg = GetChangeCurrentRevision(host, change)
778 if not jmsg:
779 raise GerritError(
780 200, 'Could not get review information for change "%s"' % change)
781 elif jmsg[0]['current_revision'] != revision:
782 raise GerritError(200, 'While resetting labels on change "%s", '
783 'a new patchset was uploaded.' % change)
Dan Jacques8d11e482016-11-15 14:25:56 -0800784
785
dimu833c94c2017-01-18 17:36:15 -0800786def CreateGerritBranch(host, project, branch, commit):
787 """
788 Create a new branch from given project and commit
789 https://gerrit-review.googlesource.com/Documentation/rest-api-projects.html#create-branch
790
791 Returns:
792 A JSON with 'ref' key
793 """
794 path = 'projects/%s/branches/%s' % (project, branch)
795 body = {'revision': commit}
796 conn = CreateHttpConn(host, path, reqtype='PUT', body=body)
797 response = ReadHttpJsonResponse(conn)
798 if response:
799 return response
800 raise GerritError(200, 'Unable to create gerrit branch')
801
802
803def GetGerritBranch(host, project, branch):
804 """
805 Get a branch from given project and commit
806 https://gerrit-review.googlesource.com/Documentation/rest-api-projects.html#get-branch
807
808 Returns:
809 A JSON object with 'revision' key
810 """
811 path = 'projects/%s/branches/%s' % (project, branch)
812 conn = CreateHttpConn(host, path, reqtype='GET')
813 response = ReadHttpJsonResponse(conn)
814 if response:
815 return response
816 raise GerritError(200, 'Unable to get gerrit branch')
817
818
Andrii Shyshkalovbb86fbb2017-03-24 14:59:28 +0100819def GetAccountDetails(host, account_id='self'):
820 """Returns details of the account.
821
822 If account_id is not given, uses magic value 'self' which corresponds to
823 whichever account user is authenticating as.
824
825 Documentation:
826 https://gerrit-review.googlesource.com/Documentation/rest-api-accounts.html#get-account
827 """
828 if account_id != 'self':
829 account_id = int(account_id)
830 conn = CreateHttpConn(host, '/accounts/%s' % account_id)
831 return ReadHttpJsonResponse(conn)
832
833
Dan Jacques8d11e482016-11-15 14:25:56 -0800834@contextlib.contextmanager
835def tempdir():
836 tdir = None
837 try:
838 tdir = tempfile.mkdtemp(suffix='gerrit_util')
839 yield tdir
840 finally:
841 if tdir:
842 gclient_utils.rmtree(tdir)