blob: be6bca6f453f9d7232ca0b160366dd6e84cc2829 [file] [log] [blame]
szager@chromium.orgb4696232013-10-16 19:45:35 +00001# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""
6Utilities for requesting information for a gerrit server via https.
7
8https://gerrit-review.googlesource.com/Documentation/rest-api.html
9"""
10
11import base64
Dan Jacques8d11e482016-11-15 14:25:56 -080012import contextlib
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +000013import cookielib
szager@chromium.orgb4696232013-10-16 19:45:35 +000014import httplib
15import json
16import logging
17import netrc
18import os
nodir@chromium.orgce32b6e2014-05-12 20:31:32 +000019import re
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +000020import socket
szager@chromium.orgf202a252014-05-27 18:55:52 +000021import stat
22import sys
Dan Jacques8d11e482016-11-15 14:25:56 -080023import tempfile
szager@chromium.orgb4696232013-10-16 19:45:35 +000024import time
25import urllib
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +000026import urlparse
szager@chromium.orgb4696232013-10-16 19:45:35 +000027from cStringIO import StringIO
28
Dan Jacques8d11e482016-11-15 14:25:56 -080029import gclient_utils
szager@chromium.orgf202a252014-05-27 18:55:52 +000030
szager@chromium.orgb4696232013-10-16 19:45:35 +000031LOGGER = logging.getLogger()
32TRY_LIMIT = 5
33
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +000034
szager@chromium.orgb4696232013-10-16 19:45:35 +000035# Controls the transport protocol used to communicate with gerrit.
36# This is parameterized primarily to enable GerritTestCase.
37GERRIT_PROTOCOL = 'https'
38
39
40class GerritError(Exception):
41 """Exception class for errors commuicating with the gerrit-on-borg service."""
42 def __init__(self, http_status, *args, **kwargs):
43 super(GerritError, self).__init__(*args, **kwargs)
44 self.http_status = http_status
45 self.message = '(%d) %s' % (self.http_status, self.message)
46
47
nodir@chromium.orgce32b6e2014-05-12 20:31:32 +000048class GerritAuthenticationError(GerritError):
49 """Exception class for authentication errors during Gerrit communication."""
50
51
szager@chromium.orgb4696232013-10-16 19:45:35 +000052def _QueryString(param_dict, first_param=None):
53 """Encodes query parameters in the key:val[+key:val...] format specified here:
54
55 https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#list-changes
56 """
57 q = [urllib.quote(first_param)] if first_param else []
58 q.extend(['%s:%s' % (key, val) for key, val in param_dict.iteritems()])
59 return '+'.join(q)
60
61
62def GetConnectionClass(protocol=None):
63 if protocol is None:
64 protocol = GERRIT_PROTOCOL
65 if protocol == 'https':
66 return httplib.HTTPSConnection
67 elif protocol == 'http':
68 return httplib.HTTPConnection
69 else:
70 raise RuntimeError(
71 "Don't know how to work with protocol '%s'" % protocol)
72
73
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +000074class Authenticator(object):
75 """Base authenticator class for authenticator implementations to subclass."""
76
77 def get_auth_header(self, host):
78 raise NotImplementedError()
79
80 @staticmethod
81 def get():
82 """Returns: (Authenticator) The identified Authenticator to use.
83
84 Probes the local system and its environment and identifies the
85 Authenticator instance to use.
86 """
87 if GceAuthenticator.is_gce():
88 return GceAuthenticator()
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +000089 return CookiesAuthenticator()
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +000090
91
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +000092class CookiesAuthenticator(Authenticator):
93 """Authenticator implementation that uses ".netrc" or ".gitcookies" for token.
94
95 Expected case for developer workstations.
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +000096 """
97
98 def __init__(self):
99 self.netrc = self._get_netrc()
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +0000100 self.gitcookies = self._get_gitcookies()
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000101
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000102 @classmethod
103 def get_new_password_message(cls, host):
104 assert not host.startswith('http')
105 # Assume *.googlesource.com pattern.
106 parts = host.split('.')
107 if not parts[0].endswith('-review'):
108 parts[0] += '-review'
109 url = 'https://%s/new-password' % ('.'.join(parts))
110 return 'You can (re)generate your credentails by visiting %s' % url
111
112 @classmethod
113 def get_netrc_path(cls):
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000114 path = '_netrc' if sys.platform.startswith('win') else '.netrc'
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000115 return os.path.expanduser(os.path.join('~', path))
116
117 @classmethod
118 def _get_netrc(cls):
Dan Jacques8d11e482016-11-15 14:25:56 -0800119 # Buffer the '.netrc' path. Use an empty file if it doesn't exist.
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000120 path = cls.get_netrc_path()
Dan Jacques8d11e482016-11-15 14:25:56 -0800121 content = ''
122 if os.path.exists(path):
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000123 st = os.stat(path)
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000124 if st.st_mode & (stat.S_IRWXG | stat.S_IRWXO):
125 print >> sys.stderr, (
126 'WARNING: netrc file %s cannot be used because its file '
127 'permissions are insecure. netrc file permissions should be '
128 '600.' % path)
Dan Jacques8d11e482016-11-15 14:25:56 -0800129 with open(path) as fd:
130 content = fd.read()
131
132 # Load the '.netrc' file. We strip comments from it because processing them
133 # can trigger a bug in Windows. See crbug.com/664664.
134 content = '\n'.join(l for l in content.splitlines()
135 if l.strip() and not l.strip().startswith('#'))
136 with tempdir() as tdir:
137 netrc_path = os.path.join(tdir, 'netrc')
138 with open(netrc_path, 'w') as fd:
139 fd.write(content)
140 os.chmod(netrc_path, (stat.S_IRUSR | stat.S_IWUSR))
141 return cls._get_netrc_from_path(netrc_path)
142
143 @classmethod
144 def _get_netrc_from_path(cls, path):
145 try:
146 return netrc.netrc(path)
147 except IOError:
148 print >> sys.stderr, 'WARNING: Could not read netrc file %s' % path
149 return netrc.netrc(os.devnull)
150 except netrc.NetrcParseError as e:
151 print >> sys.stderr, ('ERROR: Cannot use netrc file %s due to a '
152 'parsing error: %s' % (path, e))
153 return netrc.netrc(os.devnull)
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000154
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000155 @classmethod
156 def get_gitcookies_path(cls):
Ravi Mistry0bfa9ad2016-11-21 12:58:31 -0500157 if os.getenv('GIT_COOKIES_PATH'):
158 return os.getenv('GIT_COOKIES_PATH')
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000159 return os.path.join(os.environ['HOME'], '.gitcookies')
160
161 @classmethod
162 def _get_gitcookies(cls):
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +0000163 gitcookies = {}
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000164 path = cls.get_gitcookies_path()
165 if not os.path.exists(path):
166 return gitcookies
167
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +0000168 try:
169 f = open(path, 'rb')
170 except IOError:
171 return gitcookies
172
173 with f:
174 for line in f:
175 try:
176 fields = line.strip().split('\t')
177 if line.strip().startswith('#') or len(fields) != 7:
178 continue
179 domain, xpath, key, value = fields[0], fields[2], fields[5], fields[6]
180 if xpath == '/' and key == 'o':
181 login, secret_token = value.split('=', 1)
182 gitcookies[domain] = (login, secret_token)
183 except (IndexError, ValueError, TypeError) as exc:
Andrii Shyshkalov5b04a572017-01-23 17:44:41 +0100184 LOGGER.warning(exc)
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +0000185
186 return gitcookies
187
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000188 def get_auth_header(self, host):
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +0000189 auth = None
190 for domain, creds in self.gitcookies.iteritems():
191 if cookielib.domain_match(host, domain):
192 auth = (creds[0], None, creds[1])
193 break
194
195 if not auth:
196 auth = self.netrc.authenticators(host)
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000197 if auth:
198 return 'Basic %s' % (base64.b64encode('%s:%s' % (auth[0], auth[2])))
199 return None
200
Andrii Shyshkalov18975322017-01-25 16:44:13 +0100201
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000202# Backwards compatibility just in case somebody imports this outside of
203# depot_tools.
204NetrcAuthenticator = CookiesAuthenticator
205
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000206
207class GceAuthenticator(Authenticator):
208 """Authenticator implementation that uses GCE metadata service for token.
209 """
210
211 _INFO_URL = 'http://metadata.google.internal'
212 _ACQUIRE_URL = ('http://metadata/computeMetadata/v1/instance/'
213 'service-accounts/default/token')
214 _ACQUIRE_HEADERS = {"Metadata-Flavor": "Google"}
215
216 _cache_is_gce = None
217 _token_cache = None
218 _token_expiration = None
219
220 @classmethod
221 def is_gce(cls):
Ravi Mistryfad941b2016-11-15 13:00:47 -0500222 if os.getenv('SKIP_GCE_AUTH_FOR_GIT'):
223 return False
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000224 if cls._cache_is_gce is None:
225 cls._cache_is_gce = cls._test_is_gce()
226 return cls._cache_is_gce
227
228 @classmethod
229 def _test_is_gce(cls):
230 # Based on https://cloud.google.com/compute/docs/metadata#runninggce
231 try:
232 resp = cls._get(cls._INFO_URL)
233 except socket.error:
234 # Could not resolve URL.
235 return False
236 return resp.getheader('Metadata-Flavor', None) == 'Google'
237
238 @staticmethod
239 def _get(url, **kwargs):
240 next_delay_sec = 1
241 for i in xrange(TRY_LIMIT):
242 if i > 0:
243 # Retry server error status codes.
244 LOGGER.info('Encountered server error; retrying after %d second(s).',
245 next_delay_sec)
246 time.sleep(next_delay_sec)
247 next_delay_sec *= 2
248
249 p = urlparse.urlparse(url)
250 c = GetConnectionClass(protocol=p.scheme)(p.netloc)
251 c.request('GET', url, **kwargs)
252 resp = c.getresponse()
253 LOGGER.debug('GET [%s] #%d/%d (%d)', url, i+1, TRY_LIMIT, resp.status)
254 if resp.status < httplib.INTERNAL_SERVER_ERROR:
255 return resp
256
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000257 @classmethod
258 def _get_token_dict(cls):
259 if cls._token_cache:
260 # If it expires within 25 seconds, refresh.
261 if cls._token_expiration < time.time() - 25:
262 return cls._token_cache
263
264 resp = cls._get(cls._ACQUIRE_URL, headers=cls._ACQUIRE_HEADERS)
265 if resp.status != httplib.OK:
266 return None
267 cls._token_cache = json.load(resp)
268 cls._token_expiration = cls._token_cache['expires_in'] + time.time()
269 return cls._token_cache
270
271 def get_auth_header(self, _host):
272 token_dict = self._get_token_dict()
273 if not token_dict:
274 return None
275 return '%(token_type)s %(access_token)s' % token_dict
276
277
szager@chromium.orgb4696232013-10-16 19:45:35 +0000278def CreateHttpConn(host, path, reqtype='GET', headers=None, body=None):
279 """Opens an https connection to a gerrit service, and sends a request."""
280 headers = headers or {}
281 bare_host = host.partition(':')[0]
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000282
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000283 auth = Authenticator.get().get_auth_header(bare_host)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000284 if auth:
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000285 headers.setdefault('Authorization', auth)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000286 else:
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000287 LOGGER.debug('No authorization found for %s.' % bare_host)
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000288
Dan Jacques6d5bcc22016-11-14 15:32:04 -0800289 url = path
290 if not url.startswith('/'):
291 url = '/' + url
292 if 'Authorization' in headers and not url.startswith('/a/'):
293 url = '/a%s' % url
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000294
szager@chromium.orgb4696232013-10-16 19:45:35 +0000295 if body:
296 body = json.JSONEncoder().encode(body)
297 headers.setdefault('Content-Type', 'application/json')
298 if LOGGER.isEnabledFor(logging.DEBUG):
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000299 LOGGER.debug('%s %s://%s%s' % (reqtype, GERRIT_PROTOCOL, host, url))
szager@chromium.orgb4696232013-10-16 19:45:35 +0000300 for key, val in headers.iteritems():
301 if key == 'Authorization':
302 val = 'HIDDEN'
303 LOGGER.debug('%s: %s' % (key, val))
304 if body:
305 LOGGER.debug(body)
306 conn = GetConnectionClass()(host)
307 conn.req_host = host
308 conn.req_params = {
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000309 'url': url,
szager@chromium.orgb4696232013-10-16 19:45:35 +0000310 'method': reqtype,
311 'headers': headers,
312 'body': body,
313 }
314 conn.request(**conn.req_params)
315 return conn
316
317
318def ReadHttpResponse(conn, expect_status=200, ignore_404=True):
319 """Reads an http response from a connection into a string buffer.
320
321 Args:
322 conn: An HTTPSConnection or HTTPConnection created by CreateHttpConn, above.
323 expect_status: Success is indicated by this status in the response.
324 ignore_404: For many requests, gerrit-on-borg will return 404 if the request
325 doesn't match the database contents. In most such cases, we
326 want the API to return None rather than raise an Exception.
327 Returns: A string buffer containing the connection's reply.
328 """
329
330 sleep_time = 0.5
331 for idx in range(TRY_LIMIT):
332 response = conn.getresponse()
nodir@chromium.orgce32b6e2014-05-12 20:31:32 +0000333
334 # Check if this is an authentication issue.
335 www_authenticate = response.getheader('www-authenticate')
336 if (response.status in (httplib.UNAUTHORIZED, httplib.FOUND) and
337 www_authenticate):
338 auth_match = re.search('realm="([^"]+)"', www_authenticate, re.I)
339 host = auth_match.group(1) if auth_match else conn.req_host
340 reason = ('Authentication failed. Please make sure your .netrc file '
341 'has credentials for %s' % host)
342 raise GerritAuthenticationError(response.status, reason)
343
szager@chromium.orgb4696232013-10-16 19:45:35 +0000344 # If response.status < 500 then the result is final; break retry loop.
345 if response.status < 500:
Andrii Shyshkalov5b04a572017-01-23 17:44:41 +0100346 LOGGER.debug('got response %d for %s %s', response.status,
347 conn.req_params['method'], conn.req_params['url'])
szager@chromium.orgb4696232013-10-16 19:45:35 +0000348 break
349 # A status >=500 is assumed to be a possible transient error; retry.
350 http_version = 'HTTP/%s' % ('1.1' if response.version == 11 else '1.0')
Andrii Shyshkalov5b04a572017-01-23 17:44:41 +0100351 LOGGER.warn('A transient error occurred while querying %s:\n'
352 '%s %s %s\n'
353 '%s %d %s',
354 conn.host, conn.req_params['method'], conn.req_params['url'],
355 http_version, http_version, response.status, response.reason)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000356 if TRY_LIMIT - idx > 1:
Andrii Shyshkalov5b04a572017-01-23 17:44:41 +0100357 LOGGER.warn('... will retry %d more times.', TRY_LIMIT - idx - 1)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000358 time.sleep(sleep_time)
359 sleep_time = sleep_time * 2
360 req_host = conn.req_host
361 req_params = conn.req_params
362 conn = GetConnectionClass()(req_host)
363 conn.req_host = req_host
364 conn.req_params = req_params
365 conn.request(**req_params)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000366 if ignore_404 and response.status == 404:
367 return StringIO()
368 if response.status != expect_status:
nodir@chromium.orga7798032014-04-30 23:40:53 +0000369 reason = '%s: %s' % (response.reason, response.read())
370 raise GerritError(response.status, reason)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000371 return StringIO(response.read())
372
373
374def ReadHttpJsonResponse(conn, expect_status=200, ignore_404=True):
375 """Parses an https response as json."""
376 fh = ReadHttpResponse(
377 conn, expect_status=expect_status, ignore_404=ignore_404)
378 # The first line of the response should always be: )]}'
379 s = fh.readline()
380 if s and s.rstrip() != ")]}'":
381 raise GerritError(200, 'Unexpected json output: %s' % s)
382 s = fh.read()
383 if not s:
384 return None
385 return json.loads(s)
386
387
388def QueryChanges(host, param_dict, first_param=None, limit=None, o_params=None,
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100389 start=None):
szager@chromium.orgb4696232013-10-16 19:45:35 +0000390 """
391 Queries a gerrit-on-borg server for changes matching query terms.
392
393 Args:
394 param_dict: A dictionary of search parameters, as documented here:
395 http://gerrit-documentation.googlecode.com/svn/Documentation/2.6/user-search.html
396 first_param: A change identifier
397 limit: Maximum number of results to return.
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100398 start: how many changes to skip (starting with the most recent)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000399 o_params: A list of additional output specifiers, as documented here:
400 https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#list-changes
401 Returns:
402 A list of json-decoded query results.
403 """
404 # Note that no attempt is made to escape special characters; YMMV.
405 if not param_dict and not first_param:
406 raise RuntimeError('QueryChanges requires search parameters')
407 path = 'changes/?q=%s' % _QueryString(param_dict, first_param)
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100408 if start:
409 path = '%s&start=%s' % (path, start)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000410 if limit:
411 path = '%s&n=%d' % (path, limit)
412 if o_params:
413 path = '%s&%s' % (path, '&'.join(['o=%s' % p for p in o_params]))
414 # Don't ignore 404; a query should always return a list, even if it's empty.
415 return ReadHttpJsonResponse(CreateHttpConn(host, path), ignore_404=False)
416
417
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000418def GenerateAllChanges(host, param_dict, first_param=None, limit=500,
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100419 o_params=None, start=None):
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000420 """
421 Queries a gerrit-on-borg server for all the changes matching the query terms.
422
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100423 WARNING: this is unreliable if a change matching the query is modified while
424 this function is being called.
425
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000426 A single query to gerrit-on-borg is limited on the number of results by the
427 limit parameter on the request (see QueryChanges) and the server maximum
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100428 limit.
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000429
430 Args:
431 param_dict, first_param: Refer to QueryChanges().
432 limit: Maximum number of requested changes per query.
433 o_params: Refer to QueryChanges().
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100434 start: Refer to QueryChanges().
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000435
436 Returns:
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100437 A generator object to the list of returned changes.
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000438 """
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100439 already_returned = set()
440 def at_most_once(cls):
441 for cl in cls:
442 if cl['_number'] not in already_returned:
443 already_returned.add(cl['_number'])
444 yield cl
445
446 start = start or 0
447 cur_start = start
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000448 more_changes = True
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100449
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000450 while more_changes:
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100451 # This will fetch changes[start..start+limit] sorted by most recently
452 # updated. Since the rank of any change in this list can be changed any time
453 # (say user posting comment), subsequent calls may overalp like this:
454 # > initial order ABCDEFGH
455 # query[0..3] => ABC
456 # > E get's updated. New order: EABCDFGH
457 # query[3..6] => CDF # C is a dup
458 # query[6..9] => GH # E is missed.
459 page = QueryChanges(host, param_dict, first_param, limit, o_params,
460 cur_start)
461 for cl in at_most_once(page):
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000462 yield cl
463
464 more_changes = [cl for cl in page if '_more_changes' in cl]
465 if len(more_changes) > 1:
466 raise GerritError(
467 200,
468 'Received %d changes with a _more_changes attribute set but should '
469 'receive at most one.' % len(more_changes))
470 if more_changes:
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100471 cur_start += len(page)
472
473 # If we paged through, query again the first page which in most circumstances
474 # will fetch all changes that were modified while this function was run.
475 if start != cur_start:
476 page = QueryChanges(host, param_dict, first_param, limit, o_params, start)
477 for cl in at_most_once(page):
478 yield cl
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000479
480
szager@chromium.orgb4696232013-10-16 19:45:35 +0000481def MultiQueryChanges(host, param_dict, change_list, limit=None, o_params=None,
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100482 start=None):
szager@chromium.orgb4696232013-10-16 19:45:35 +0000483 """Initiate a query composed of multiple sets of query parameters."""
484 if not change_list:
485 raise RuntimeError(
486 "MultiQueryChanges requires a list of change numbers/id's")
487 q = ['q=%s' % '+OR+'.join([urllib.quote(str(x)) for x in change_list])]
488 if param_dict:
489 q.append(_QueryString(param_dict))
490 if limit:
491 q.append('n=%d' % limit)
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100492 if start:
493 q.append('S=%s' % start)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000494 if o_params:
495 q.extend(['o=%s' % p for p in o_params])
496 path = 'changes/?%s' % '&'.join(q)
497 try:
498 result = ReadHttpJsonResponse(CreateHttpConn(host, path), ignore_404=False)
499 except GerritError as e:
500 msg = '%s:\n%s' % (e.message, path)
501 raise GerritError(e.http_status, msg)
502 return result
503
504
505def GetGerritFetchUrl(host):
506 """Given a gerrit host name returns URL of a gerrit instance to fetch from."""
507 return '%s://%s/' % (GERRIT_PROTOCOL, host)
508
509
510def GetChangePageUrl(host, change_number):
511 """Given a gerrit host name and change number, return change page url."""
512 return '%s://%s/#/c/%d/' % (GERRIT_PROTOCOL, host, change_number)
513
514
515def GetChangeUrl(host, change):
516 """Given a gerrit host name and change id, return an url for the change."""
517 return '%s://%s/a/changes/%s' % (GERRIT_PROTOCOL, host, change)
518
519
520def GetChange(host, change):
521 """Query a gerrit server for information about a single change."""
522 path = 'changes/%s' % change
523 return ReadHttpJsonResponse(CreateHttpConn(host, path))
524
525
Andrii Shyshkalovc6c8b4c2016-11-09 20:51:20 +0100526def GetChangeDetail(host, change, o_params=None, ignore_404=True):
szager@chromium.orgb4696232013-10-16 19:45:35 +0000527 """Query a gerrit server for extended information about a single change."""
Andrii Shyshkalovc6c8b4c2016-11-09 20:51:20 +0100528 # TODO(tandrii): cahnge ignore_404 to False by default.
szager@chromium.orgb4696232013-10-16 19:45:35 +0000529 path = 'changes/%s/detail' % change
530 if o_params:
531 path += '?%s' % '&'.join(['o=%s' % p for p in o_params])
Andrii Shyshkalovc6c8b4c2016-11-09 20:51:20 +0100532 return ReadHttpJsonResponse(CreateHttpConn(host, path), ignore_404=ignore_404)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000533
534
agable32978d92016-11-01 12:55:02 -0700535def GetChangeCommit(host, change, revision='current'):
536 """Query a gerrit server for a revision associated with a change."""
537 path = 'changes/%s/revisions/%s/commit?links' % (change, revision)
538 return ReadHttpJsonResponse(CreateHttpConn(host, path))
539
540
szager@chromium.orgb4696232013-10-16 19:45:35 +0000541def GetChangeCurrentRevision(host, change):
542 """Get information about the latest revision for a given change."""
543 return QueryChanges(host, {}, change, o_params=('CURRENT_REVISION',))
544
545
546def GetChangeRevisions(host, change):
547 """Get information about all revisions associated with a change."""
548 return QueryChanges(host, {}, change, o_params=('ALL_REVISIONS',))
549
550
551def GetChangeReview(host, change, revision=None):
552 """Get the current review information for a change."""
553 if not revision:
554 jmsg = GetChangeRevisions(host, change)
555 if not jmsg:
556 return None
557 elif len(jmsg) > 1:
558 raise GerritError(200, 'Multiple changes found for ChangeId %s.' % change)
559 revision = jmsg[0]['current_revision']
560 path = 'changes/%s/revisions/%s/review'
561 return ReadHttpJsonResponse(CreateHttpConn(host, path))
562
563
564def AbandonChange(host, change, msg=''):
565 """Abandon a gerrit change."""
566 path = 'changes/%s/abandon' % change
tandrii@chromium.orgc7da66a2016-03-24 09:52:24 +0000567 body = {'message': msg} if msg else {}
szager@chromium.orgb4696232013-10-16 19:45:35 +0000568 conn = CreateHttpConn(host, path, reqtype='POST', body=body)
569 return ReadHttpJsonResponse(conn, ignore_404=False)
570
571
572def RestoreChange(host, change, msg=''):
573 """Restore a previously abandoned change."""
574 path = 'changes/%s/restore' % change
tandrii@chromium.orgc7da66a2016-03-24 09:52:24 +0000575 body = {'message': msg} if msg else {}
szager@chromium.orgb4696232013-10-16 19:45:35 +0000576 conn = CreateHttpConn(host, path, reqtype='POST', body=body)
577 return ReadHttpJsonResponse(conn, ignore_404=False)
578
579
580def SubmitChange(host, change, wait_for_merge=True):
581 """Submits a gerrit change via Gerrit."""
582 path = 'changes/%s/submit' % change
583 body = {'wait_for_merge': wait_for_merge}
584 conn = CreateHttpConn(host, path, reqtype='POST', body=body)
585 return ReadHttpJsonResponse(conn, ignore_404=False)
586
587
dsansomee2d6fd92016-09-08 00:10:47 -0700588def HasPendingChangeEdit(host, change):
589 conn = CreateHttpConn(host, 'changes/%s/edit' % change)
590 try:
591 ReadHttpResponse(conn, ignore_404=False)
592 except GerritError as e:
593 # On success, gerrit returns status 204; anything else is an error.
594 if e.http_status != 204:
595 raise
596 return False
597 else:
598 return True
599
600
601def DeletePendingChangeEdit(host, change):
602 conn = CreateHttpConn(host, 'changes/%s/edit' % change, reqtype='DELETE')
603 try:
604 ReadHttpResponse(conn, ignore_404=False)
605 except GerritError as e:
606 # On success, gerrit returns status 204; if the edit was already deleted it
607 # returns 404. Anything else is an error.
608 if e.http_status not in (204, 404):
609 raise
610
611
Andrii Shyshkalovea4fc832016-12-01 14:53:23 +0100612def SetCommitMessage(host, change, description, notify='ALL'):
scottmg@chromium.org6d1266e2016-04-26 11:12:26 +0000613 """Updates a commit message."""
scottmg@chromium.org6d1266e2016-04-26 11:12:26 +0000614 try:
Aaron Gablee9373d62016-12-13 12:28:45 -0800615 assert notify in ('ALL', 'NONE')
616 # First, edit the commit message in a draft.
617 path = 'changes/%s/edit:message' % change
618 body = {'message': description}
619 conn = CreateHttpConn(host, path, reqtype='PUT', body=body)
620 try:
621 ReadHttpResponse(conn, ignore_404=False)
622 except GerritError as e:
623 # On success, gerrit returns status 204; anything else is an error.
624 if e.http_status != 204:
625 raise
626 else:
627 raise GerritError(
628 'Unexpectedly received a 200 http status while editing message in '
629 'change %s' % change)
scottmg@chromium.org6d1266e2016-04-26 11:12:26 +0000630
Aaron Gablee9373d62016-12-13 12:28:45 -0800631 # And then publish it.
632 path = 'changes/%s/edit:publish' % change
633 conn = CreateHttpConn(host, path, reqtype='POST', body={'notify': notify})
634 try:
635 ReadHttpResponse(conn, ignore_404=False)
636 except GerritError as e:
637 # On success, gerrit returns status 204; anything else is an error.
638 if e.http_status != 204:
639 raise
640 else:
641 raise GerritError(
642 'Unexpectedly received a 200 http status while publishing message '
643 'change in %s' % change)
644 except (GerritError, KeyboardInterrupt) as e:
645 # Something went wrong with one of the two calls, so we want to clean up
646 # after ourselves before returning.
647 try:
648 DeletePendingChangeEdit(host, change)
649 except GerritError:
650 LOGGER.error('Encountered error while cleaning up after failed attempt '
651 'to set the CL description. You may have to delete the '
652 'pending change edit yourself in the web UI.')
653 raise e
scottmg@chromium.org6d1266e2016-04-26 11:12:26 +0000654
655
szager@chromium.orgb4696232013-10-16 19:45:35 +0000656def GetReviewers(host, change):
657 """Get information about all reviewers attached to a change."""
658 path = 'changes/%s/reviewers' % change
659 return ReadHttpJsonResponse(CreateHttpConn(host, path))
660
661
662def GetReview(host, change, revision):
663 """Get review information about a specific revision of a change."""
664 path = 'changes/%s/revisions/%s/review' % (change, revision)
665 return ReadHttpJsonResponse(CreateHttpConn(host, path))
666
667
Aaron Gable59f48512017-01-12 10:54:46 -0800668def AddReviewers(host, change, add=None, is_reviewer=True, notify=True):
szager@chromium.orgb4696232013-10-16 19:45:35 +0000669 """Add reviewers to a change."""
Aaron Gabledf86e302016-11-08 10:48:03 -0800670 errors = None
szager@chromium.orgb4696232013-10-16 19:45:35 +0000671 if not add:
Aaron Gabledf86e302016-11-08 10:48:03 -0800672 return None
szager@chromium.orgb4696232013-10-16 19:45:35 +0000673 if isinstance(add, basestring):
674 add = (add,)
675 path = 'changes/%s/reviewers' % change
676 for r in add:
Aaron Gabledf86e302016-11-08 10:48:03 -0800677 state = 'REVIEWER' if is_reviewer else 'CC'
Aaron Gable59f48512017-01-12 10:54:46 -0800678 notify = 'ALL' if notify else 'NONE'
tandrii88189772016-09-29 04:29:57 -0700679 body = {
680 'reviewer': r,
Aaron Gabledf86e302016-11-08 10:48:03 -0800681 'state': state,
Aaron Gable59f48512017-01-12 10:54:46 -0800682 'notify': notify,
tandrii88189772016-09-29 04:29:57 -0700683 }
Aaron Gabledf86e302016-11-08 10:48:03 -0800684 try:
685 conn = CreateHttpConn(host, path, reqtype='POST', body=body)
686 _ = ReadHttpJsonResponse(conn, ignore_404=False)
687 except GerritError as e:
688 if e.http_status == 422: # "Unprocessable Entity"
Aaron Gableb7cb65a2017-03-14 11:39:41 -0700689 LOGGER.warn('Note: "%s" not added as a %s' % (r, state.lower()))
Aaron Gabledf86e302016-11-08 10:48:03 -0800690 errors = True
691 else:
692 raise
693 return errors
szager@chromium.orgb4696232013-10-16 19:45:35 +0000694
695
696def RemoveReviewers(host, change, remove=None):
697 """Remove reveiewers from a change."""
698 if not remove:
699 return
700 if isinstance(remove, basestring):
701 remove = (remove,)
702 for r in remove:
703 path = 'changes/%s/reviewers/%s' % (change, r)
704 conn = CreateHttpConn(host, path, reqtype='DELETE')
705 try:
706 ReadHttpResponse(conn, ignore_404=False)
707 except GerritError as e:
708 # On success, gerrit returns status 204; anything else is an error.
709 if e.http_status != 204:
710 raise
711 else:
712 raise GerritError(
713 'Unexpectedly received a 200 http status while deleting reviewer "%s"'
714 ' from change %s' % (r, change))
715
716
717def SetReview(host, change, msg=None, labels=None, notify=None):
718 """Set labels and/or add a message to a code review."""
719 if not msg and not labels:
720 return
721 path = 'changes/%s/revisions/current/review' % change
722 body = {}
723 if msg:
724 body['message'] = msg
725 if labels:
726 body['labels'] = labels
727 if notify:
728 body['notify'] = notify
729 conn = CreateHttpConn(host, path, reqtype='POST', body=body)
730 response = ReadHttpJsonResponse(conn)
731 if labels:
732 for key, val in labels.iteritems():
733 if ('labels' not in response or key not in response['labels'] or
734 int(response['labels'][key] != int(val))):
735 raise GerritError(200, 'Unable to set "%s" label on change %s.' % (
736 key, change))
737
738
739def ResetReviewLabels(host, change, label, value='0', message=None,
740 notify=None):
741 """Reset the value of a given label for all reviewers on a change."""
742 # This is tricky, because we want to work on the "current revision", but
743 # there's always the risk that "current revision" will change in between
744 # API calls. So, we check "current revision" at the beginning and end; if
745 # it has changed, raise an exception.
746 jmsg = GetChangeCurrentRevision(host, change)
747 if not jmsg:
748 raise GerritError(
749 200, 'Could not get review information for change "%s"' % change)
750 value = str(value)
751 revision = jmsg[0]['current_revision']
752 path = 'changes/%s/revisions/%s/review' % (change, revision)
753 message = message or (
754 '%s label set to %s programmatically.' % (label, value))
755 jmsg = GetReview(host, change, revision)
756 if not jmsg:
757 raise GerritError(200, 'Could not get review information for revison %s '
758 'of change %s' % (revision, change))
759 for review in jmsg.get('labels', {}).get(label, {}).get('all', []):
760 if str(review.get('value', value)) != value:
761 body = {
762 'message': message,
763 'labels': {label: value},
764 'on_behalf_of': review['_account_id'],
765 }
766 if notify:
767 body['notify'] = notify
768 conn = CreateHttpConn(
769 host, path, reqtype='POST', body=body)
770 response = ReadHttpJsonResponse(conn)
771 if str(response['labels'][label]) != value:
772 username = review.get('email', jmsg.get('name', ''))
773 raise GerritError(200, 'Unable to set %s label for user "%s"'
774 ' on change %s.' % (label, username, change))
775 jmsg = GetChangeCurrentRevision(host, change)
776 if not jmsg:
777 raise GerritError(
778 200, 'Could not get review information for change "%s"' % change)
779 elif jmsg[0]['current_revision'] != revision:
780 raise GerritError(200, 'While resetting labels on change "%s", '
781 'a new patchset was uploaded.' % change)
Dan Jacques8d11e482016-11-15 14:25:56 -0800782
783
dimu833c94c2017-01-18 17:36:15 -0800784def CreateGerritBranch(host, project, branch, commit):
785 """
786 Create a new branch from given project and commit
787 https://gerrit-review.googlesource.com/Documentation/rest-api-projects.html#create-branch
788
789 Returns:
790 A JSON with 'ref' key
791 """
792 path = 'projects/%s/branches/%s' % (project, branch)
793 body = {'revision': commit}
794 conn = CreateHttpConn(host, path, reqtype='PUT', body=body)
795 response = ReadHttpJsonResponse(conn)
796 if response:
797 return response
798 raise GerritError(200, 'Unable to create gerrit branch')
799
800
801def GetGerritBranch(host, project, branch):
802 """
803 Get a branch from given project and commit
804 https://gerrit-review.googlesource.com/Documentation/rest-api-projects.html#get-branch
805
806 Returns:
807 A JSON object with 'revision' key
808 """
809 path = 'projects/%s/branches/%s' % (project, branch)
810 conn = CreateHttpConn(host, path, reqtype='GET')
811 response = ReadHttpJsonResponse(conn)
812 if response:
813 return response
814 raise GerritError(200, 'Unable to get gerrit branch')
815
816
Dan Jacques8d11e482016-11-15 14:25:56 -0800817@contextlib.contextmanager
818def tempdir():
819 tdir = None
820 try:
821 tdir = tempfile.mkdtemp(suffix='gerrit_util')
822 yield tdir
823 finally:
824 if tdir:
825 gclient_utils.rmtree(tdir)