blob: 46bf794774296e2009fab2f9581ea64958631bcf [file] [log] [blame]
szager@chromium.orgb4696232013-10-16 19:45:35 +00001# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""
6Utilities for requesting information for a gerrit server via https.
7
8https://gerrit-review.googlesource.com/Documentation/rest-api.html
9"""
10
11import base64
Dan Jacques8d11e482016-11-15 14:25:56 -080012import contextlib
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +000013import cookielib
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +010014import httplib # Still used for its constants.
szager@chromium.orgb4696232013-10-16 19:45:35 +000015import json
16import logging
17import netrc
18import os
nodir@chromium.orgce32b6e2014-05-12 20:31:32 +000019import re
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +000020import socket
szager@chromium.orgf202a252014-05-27 18:55:52 +000021import stat
22import sys
Dan Jacques8d11e482016-11-15 14:25:56 -080023import tempfile
szager@chromium.orgb4696232013-10-16 19:45:35 +000024import time
25import urllib
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +000026import urlparse
szager@chromium.orgb4696232013-10-16 19:45:35 +000027from cStringIO import StringIO
28
Dan Jacques8d11e482016-11-15 14:25:56 -080029import gclient_utils
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +010030from third_party import httplib2
szager@chromium.orgf202a252014-05-27 18:55:52 +000031
szager@chromium.orgb4696232013-10-16 19:45:35 +000032LOGGER = logging.getLogger()
33TRY_LIMIT = 5
34
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +000035
szager@chromium.orgb4696232013-10-16 19:45:35 +000036# Controls the transport protocol used to communicate with gerrit.
37# This is parameterized primarily to enable GerritTestCase.
38GERRIT_PROTOCOL = 'https'
39
40
41class GerritError(Exception):
42 """Exception class for errors commuicating with the gerrit-on-borg service."""
43 def __init__(self, http_status, *args, **kwargs):
44 super(GerritError, self).__init__(*args, **kwargs)
45 self.http_status = http_status
46 self.message = '(%d) %s' % (self.http_status, self.message)
47
48
nodir@chromium.orgce32b6e2014-05-12 20:31:32 +000049class GerritAuthenticationError(GerritError):
50 """Exception class for authentication errors during Gerrit communication."""
51
52
Michael Achenbach6fbf12f2017-07-06 10:54:11 +020053def _QueryString(params, first_param=None):
szager@chromium.orgb4696232013-10-16 19:45:35 +000054 """Encodes query parameters in the key:val[+key:val...] format specified here:
55
56 https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#list-changes
57 """
58 q = [urllib.quote(first_param)] if first_param else []
Michael Achenbach6fbf12f2017-07-06 10:54:11 +020059 q.extend(['%s:%s' % (key, val) for key, val in params])
szager@chromium.orgb4696232013-10-16 19:45:35 +000060 return '+'.join(q)
61
62
Aaron Gabled2db5a22017-03-24 14:14:15 -070063def GetConnectionObject(protocol=None):
szager@chromium.orgb4696232013-10-16 19:45:35 +000064 if protocol is None:
65 protocol = GERRIT_PROTOCOL
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +010066 if protocol in ('http', 'https'):
Aaron Gabled2db5a22017-03-24 14:14:15 -070067 return httplib2.Http()
szager@chromium.orgb4696232013-10-16 19:45:35 +000068 else:
69 raise RuntimeError(
70 "Don't know how to work with protocol '%s'" % protocol)
71
72
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +000073class Authenticator(object):
74 """Base authenticator class for authenticator implementations to subclass."""
75
76 def get_auth_header(self, host):
77 raise NotImplementedError()
78
79 @staticmethod
80 def get():
81 """Returns: (Authenticator) The identified Authenticator to use.
82
83 Probes the local system and its environment and identifies the
84 Authenticator instance to use.
85 """
86 if GceAuthenticator.is_gce():
87 return GceAuthenticator()
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +000088 return CookiesAuthenticator()
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +000089
90
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +000091class CookiesAuthenticator(Authenticator):
92 """Authenticator implementation that uses ".netrc" or ".gitcookies" for token.
93
94 Expected case for developer workstations.
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +000095 """
96
97 def __init__(self):
98 self.netrc = self._get_netrc()
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +000099 self.gitcookies = self._get_gitcookies()
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000100
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000101 @classmethod
102 def get_new_password_message(cls, host):
103 assert not host.startswith('http')
104 # Assume *.googlesource.com pattern.
105 parts = host.split('.')
106 if not parts[0].endswith('-review'):
107 parts[0] += '-review'
108 url = 'https://%s/new-password' % ('.'.join(parts))
Andrii Shyshkalov0a0b0672017-03-16 16:27:48 +0100109 return 'You can (re)generate your credentials by visiting %s' % url
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000110
111 @classmethod
112 def get_netrc_path(cls):
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000113 path = '_netrc' if sys.platform.startswith('win') else '.netrc'
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000114 return os.path.expanduser(os.path.join('~', path))
115
116 @classmethod
117 def _get_netrc(cls):
Dan Jacques8d11e482016-11-15 14:25:56 -0800118 # Buffer the '.netrc' path. Use an empty file if it doesn't exist.
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000119 path = cls.get_netrc_path()
Dan Jacques8d11e482016-11-15 14:25:56 -0800120 content = ''
121 if os.path.exists(path):
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000122 st = os.stat(path)
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000123 if st.st_mode & (stat.S_IRWXG | stat.S_IRWXO):
124 print >> sys.stderr, (
125 'WARNING: netrc file %s cannot be used because its file '
126 'permissions are insecure. netrc file permissions should be '
127 '600.' % path)
Dan Jacques8d11e482016-11-15 14:25:56 -0800128 with open(path) as fd:
129 content = fd.read()
130
131 # Load the '.netrc' file. We strip comments from it because processing them
132 # can trigger a bug in Windows. See crbug.com/664664.
133 content = '\n'.join(l for l in content.splitlines()
134 if l.strip() and not l.strip().startswith('#'))
135 with tempdir() as tdir:
136 netrc_path = os.path.join(tdir, 'netrc')
137 with open(netrc_path, 'w') as fd:
138 fd.write(content)
139 os.chmod(netrc_path, (stat.S_IRUSR | stat.S_IWUSR))
140 return cls._get_netrc_from_path(netrc_path)
141
142 @classmethod
143 def _get_netrc_from_path(cls, path):
144 try:
145 return netrc.netrc(path)
146 except IOError:
147 print >> sys.stderr, 'WARNING: Could not read netrc file %s' % path
148 return netrc.netrc(os.devnull)
149 except netrc.NetrcParseError as e:
150 print >> sys.stderr, ('ERROR: Cannot use netrc file %s due to a '
151 'parsing error: %s' % (path, e))
152 return netrc.netrc(os.devnull)
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000153
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000154 @classmethod
155 def get_gitcookies_path(cls):
Ravi Mistry0bfa9ad2016-11-21 12:58:31 -0500156 if os.getenv('GIT_COOKIES_PATH'):
157 return os.getenv('GIT_COOKIES_PATH')
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000158 return os.path.join(os.environ['HOME'], '.gitcookies')
159
160 @classmethod
161 def _get_gitcookies(cls):
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +0000162 gitcookies = {}
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000163 path = cls.get_gitcookies_path()
164 if not os.path.exists(path):
165 return gitcookies
166
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +0000167 try:
168 f = open(path, 'rb')
169 except IOError:
170 return gitcookies
171
172 with f:
173 for line in f:
174 try:
175 fields = line.strip().split('\t')
176 if line.strip().startswith('#') or len(fields) != 7:
177 continue
178 domain, xpath, key, value = fields[0], fields[2], fields[5], fields[6]
179 if xpath == '/' and key == 'o':
180 login, secret_token = value.split('=', 1)
181 gitcookies[domain] = (login, secret_token)
182 except (IndexError, ValueError, TypeError) as exc:
Andrii Shyshkalov5b04a572017-01-23 17:44:41 +0100183 LOGGER.warning(exc)
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +0000184
185 return gitcookies
186
Andrii Shyshkalovbb86fbb2017-03-24 14:59:28 +0100187 def _get_auth_for_host(self, host):
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +0000188 for domain, creds in self.gitcookies.iteritems():
189 if cookielib.domain_match(host, domain):
Andrii Shyshkalovbb86fbb2017-03-24 14:59:28 +0100190 return (creds[0], None, creds[1])
191 return self.netrc.authenticators(host)
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +0000192
Andrii Shyshkalovbb86fbb2017-03-24 14:59:28 +0100193 def get_auth_header(self, host):
194 auth = self._get_auth_for_host(host)
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000195 if auth:
196 return 'Basic %s' % (base64.b64encode('%s:%s' % (auth[0], auth[2])))
197 return None
198
Andrii Shyshkalovbb86fbb2017-03-24 14:59:28 +0100199 def get_auth_email(self, host):
200 """Best effort parsing of email to be used for auth for the given host."""
201 auth = self._get_auth_for_host(host)
202 if not auth:
203 return None
204 login = auth[0]
205 # login typically looks like 'git-xxx.example.com'
206 if not login.startswith('git-') or '.' not in login:
207 return None
208 username, domain = login[len('git-'):].split('.', 1)
209 return '%s@%s' % (username, domain)
210
Andrii Shyshkalov18975322017-01-25 16:44:13 +0100211
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000212# Backwards compatibility just in case somebody imports this outside of
213# depot_tools.
214NetrcAuthenticator = CookiesAuthenticator
215
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000216
217class GceAuthenticator(Authenticator):
218 """Authenticator implementation that uses GCE metadata service for token.
219 """
220
221 _INFO_URL = 'http://metadata.google.internal'
222 _ACQUIRE_URL = ('http://metadata/computeMetadata/v1/instance/'
223 'service-accounts/default/token')
224 _ACQUIRE_HEADERS = {"Metadata-Flavor": "Google"}
225
226 _cache_is_gce = None
227 _token_cache = None
228 _token_expiration = None
229
230 @classmethod
231 def is_gce(cls):
Ravi Mistryfad941b2016-11-15 13:00:47 -0500232 if os.getenv('SKIP_GCE_AUTH_FOR_GIT'):
233 return False
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000234 if cls._cache_is_gce is None:
235 cls._cache_is_gce = cls._test_is_gce()
236 return cls._cache_is_gce
237
238 @classmethod
239 def _test_is_gce(cls):
240 # Based on https://cloud.google.com/compute/docs/metadata#runninggce
241 try:
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100242 resp, _ = cls._get(cls._INFO_URL)
243 except (socket.error, httplib2.ServerNotFoundError):
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000244 # Could not resolve URL.
245 return False
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100246 return resp.get('metadata-flavor') == 'Google'
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000247
248 @staticmethod
249 def _get(url, **kwargs):
250 next_delay_sec = 1
251 for i in xrange(TRY_LIMIT):
252 if i > 0:
253 # Retry server error status codes.
254 LOGGER.info('Encountered server error; retrying after %d second(s).',
255 next_delay_sec)
256 time.sleep(next_delay_sec)
257 next_delay_sec *= 2
258
259 p = urlparse.urlparse(url)
Aaron Gabled2db5a22017-03-24 14:14:15 -0700260 c = GetConnectionObject(protocol=p.scheme)
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100261 resp, contents = c.request(url, 'GET', **kwargs)
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000262 LOGGER.debug('GET [%s] #%d/%d (%d)', url, i+1, TRY_LIMIT, resp.status)
263 if resp.status < httplib.INTERNAL_SERVER_ERROR:
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100264 return (resp, contents)
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000265
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000266 @classmethod
267 def _get_token_dict(cls):
268 if cls._token_cache:
269 # If it expires within 25 seconds, refresh.
270 if cls._token_expiration < time.time() - 25:
271 return cls._token_cache
272
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100273 resp, contents = cls._get(cls._ACQUIRE_URL, headers=cls._ACQUIRE_HEADERS)
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000274 if resp.status != httplib.OK:
275 return None
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100276 cls._token_cache = json.loads(contents)
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000277 cls._token_expiration = cls._token_cache['expires_in'] + time.time()
278 return cls._token_cache
279
280 def get_auth_header(self, _host):
281 token_dict = self._get_token_dict()
282 if not token_dict:
283 return None
284 return '%(token_type)s %(access_token)s' % token_dict
285
286
szager@chromium.orgb4696232013-10-16 19:45:35 +0000287def CreateHttpConn(host, path, reqtype='GET', headers=None, body=None):
288 """Opens an https connection to a gerrit service, and sends a request."""
289 headers = headers or {}
290 bare_host = host.partition(':')[0]
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000291
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000292 auth = Authenticator.get().get_auth_header(bare_host)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000293 if auth:
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000294 headers.setdefault('Authorization', auth)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000295 else:
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000296 LOGGER.debug('No authorization found for %s.' % bare_host)
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000297
Dan Jacques6d5bcc22016-11-14 15:32:04 -0800298 url = path
299 if not url.startswith('/'):
300 url = '/' + url
301 if 'Authorization' in headers and not url.startswith('/a/'):
302 url = '/a%s' % url
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000303
szager@chromium.orgb4696232013-10-16 19:45:35 +0000304 if body:
305 body = json.JSONEncoder().encode(body)
306 headers.setdefault('Content-Type', 'application/json')
307 if LOGGER.isEnabledFor(logging.DEBUG):
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000308 LOGGER.debug('%s %s://%s%s' % (reqtype, GERRIT_PROTOCOL, host, url))
szager@chromium.orgb4696232013-10-16 19:45:35 +0000309 for key, val in headers.iteritems():
310 if key == 'Authorization':
311 val = 'HIDDEN'
312 LOGGER.debug('%s: %s' % (key, val))
313 if body:
314 LOGGER.debug(body)
Aaron Gabled2db5a22017-03-24 14:14:15 -0700315 conn = GetConnectionObject()
szager@chromium.orgb4696232013-10-16 19:45:35 +0000316 conn.req_host = host
317 conn.req_params = {
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100318 'uri': urlparse.urljoin('%s://%s' % (GERRIT_PROTOCOL, host), url),
szager@chromium.orgb4696232013-10-16 19:45:35 +0000319 'method': reqtype,
320 'headers': headers,
321 'body': body,
322 }
szager@chromium.orgb4696232013-10-16 19:45:35 +0000323 return conn
324
325
Aaron Gable6f5a8d92017-04-18 14:49:05 -0700326def ReadHttpResponse(conn, accept_statuses=frozenset([200])):
szager@chromium.orgb4696232013-10-16 19:45:35 +0000327 """Reads an http response from a connection into a string buffer.
328
329 Args:
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100330 conn: An Http object created by CreateHttpConn above.
Aaron Gable6f5a8d92017-04-18 14:49:05 -0700331 accept_statuses: Treat any of these statuses as success. Default: [200]
332 Common additions include 204, 400, and 404.
szager@chromium.orgb4696232013-10-16 19:45:35 +0000333 Returns: A string buffer containing the connection's reply.
334 """
szager@chromium.orgb4696232013-10-16 19:45:35 +0000335 sleep_time = 0.5
336 for idx in range(TRY_LIMIT):
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100337 response, contents = conn.request(**conn.req_params)
nodir@chromium.orgce32b6e2014-05-12 20:31:32 +0000338
339 # Check if this is an authentication issue.
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100340 www_authenticate = response.get('www-authenticate')
nodir@chromium.orgce32b6e2014-05-12 20:31:32 +0000341 if (response.status in (httplib.UNAUTHORIZED, httplib.FOUND) and
342 www_authenticate):
343 auth_match = re.search('realm="([^"]+)"', www_authenticate, re.I)
344 host = auth_match.group(1) if auth_match else conn.req_host
Aaron Gable19ee16c2017-04-18 11:56:35 -0700345 reason = ('Authentication failed. Please make sure your .gitcookies file '
nodir@chromium.orgce32b6e2014-05-12 20:31:32 +0000346 'has credentials for %s' % host)
347 raise GerritAuthenticationError(response.status, reason)
348
szager@chromium.orgb4696232013-10-16 19:45:35 +0000349 # If response.status < 500 then the result is final; break retry loop.
Aaron Gable62ca9602017-05-19 17:24:52 -0700350 # If the response is 404, it might be because of replication lag, so
351 # keep trying anyway.
352 if response.status < 500 and response.status != 404:
Andrii Shyshkalov5b04a572017-01-23 17:44:41 +0100353 LOGGER.debug('got response %d for %s %s', response.status,
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100354 conn.req_params['method'], conn.req_params['uri'])
szager@chromium.orgb4696232013-10-16 19:45:35 +0000355 break
356 # A status >=500 is assumed to be a possible transient error; retry.
357 http_version = 'HTTP/%s' % ('1.1' if response.version == 11 else '1.0')
Andrii Shyshkalov5b04a572017-01-23 17:44:41 +0100358 LOGGER.warn('A transient error occurred while querying %s:\n'
359 '%s %s %s\n'
360 '%s %d %s',
Aaron Gable20d2cbb2017-04-25 15:04:05 -0700361 conn.req_host, conn.req_params['method'],
362 conn.req_params['uri'],
Andrii Shyshkalov5b04a572017-01-23 17:44:41 +0100363 http_version, http_version, response.status, response.reason)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000364 if TRY_LIMIT - idx > 1:
Andrii Shyshkalov5b04a572017-01-23 17:44:41 +0100365 LOGGER.warn('... will retry %d more times.', TRY_LIMIT - idx - 1)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000366 time.sleep(sleep_time)
367 sleep_time = sleep_time * 2
Aaron Gable19ee16c2017-04-18 11:56:35 -0700368 if response.status not in accept_statuses:
Andrii Shyshkalov4956f792017-04-10 14:28:38 +0200369 if response.status in (401, 403):
370 print('Your Gerrit credentials might be misconfigured. Try: \n'
371 ' git cl creds-check')
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100372 reason = '%s: %s' % (response.reason, contents)
nodir@chromium.orga7798032014-04-30 23:40:53 +0000373 raise GerritError(response.status, reason)
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100374 return StringIO(contents)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000375
376
Aaron Gable6f5a8d92017-04-18 14:49:05 -0700377def ReadHttpJsonResponse(conn, accept_statuses=frozenset([200])):
szager@chromium.orgb4696232013-10-16 19:45:35 +0000378 """Parses an https response as json."""
Aaron Gable19ee16c2017-04-18 11:56:35 -0700379 fh = ReadHttpResponse(conn, accept_statuses)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000380 # The first line of the response should always be: )]}'
381 s = fh.readline()
382 if s and s.rstrip() != ")]}'":
383 raise GerritError(200, 'Unexpected json output: %s' % s)
384 s = fh.read()
385 if not s:
386 return None
387 return json.loads(s)
388
389
Michael Achenbach6fbf12f2017-07-06 10:54:11 +0200390def QueryChanges(host, params, first_param=None, limit=None, o_params=None,
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100391 start=None):
szager@chromium.orgb4696232013-10-16 19:45:35 +0000392 """
393 Queries a gerrit-on-borg server for changes matching query terms.
394
395 Args:
Michael Achenbach6fbf12f2017-07-06 10:54:11 +0200396 params: A list of key:value pairs for search parameters, as documented
397 here (e.g. ('is', 'owner') for a parameter 'is:owner'):
398 https://gerrit-review.googlesource.com/Documentation/user-search.html#search-operators
szager@chromium.orgb4696232013-10-16 19:45:35 +0000399 first_param: A change identifier
400 limit: Maximum number of results to return.
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100401 start: how many changes to skip (starting with the most recent)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000402 o_params: A list of additional output specifiers, as documented here:
403 https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#list-changes
404 Returns:
405 A list of json-decoded query results.
406 """
407 # Note that no attempt is made to escape special characters; YMMV.
Michael Achenbach6fbf12f2017-07-06 10:54:11 +0200408 if not params and not first_param:
szager@chromium.orgb4696232013-10-16 19:45:35 +0000409 raise RuntimeError('QueryChanges requires search parameters')
Michael Achenbach6fbf12f2017-07-06 10:54:11 +0200410 path = 'changes/?q=%s' % _QueryString(params, first_param)
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100411 if start:
412 path = '%s&start=%s' % (path, start)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000413 if limit:
414 path = '%s&n=%d' % (path, limit)
415 if o_params:
416 path = '%s&%s' % (path, '&'.join(['o=%s' % p for p in o_params]))
Aaron Gable6f5a8d92017-04-18 14:49:05 -0700417 return ReadHttpJsonResponse(CreateHttpConn(host, path))
szager@chromium.orgb4696232013-10-16 19:45:35 +0000418
419
Michael Achenbach6fbf12f2017-07-06 10:54:11 +0200420def GenerateAllChanges(host, params, first_param=None, limit=500,
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100421 o_params=None, start=None):
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000422 """
423 Queries a gerrit-on-borg server for all the changes matching the query terms.
424
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100425 WARNING: this is unreliable if a change matching the query is modified while
426 this function is being called.
427
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000428 A single query to gerrit-on-borg is limited on the number of results by the
429 limit parameter on the request (see QueryChanges) and the server maximum
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100430 limit.
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000431
432 Args:
Michael Achenbach6fbf12f2017-07-06 10:54:11 +0200433 params, first_param: Refer to QueryChanges().
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000434 limit: Maximum number of requested changes per query.
435 o_params: Refer to QueryChanges().
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100436 start: Refer to QueryChanges().
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000437
438 Returns:
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100439 A generator object to the list of returned changes.
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000440 """
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100441 already_returned = set()
442 def at_most_once(cls):
443 for cl in cls:
444 if cl['_number'] not in already_returned:
445 already_returned.add(cl['_number'])
446 yield cl
447
448 start = start or 0
449 cur_start = start
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000450 more_changes = True
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100451
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000452 while more_changes:
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100453 # This will fetch changes[start..start+limit] sorted by most recently
454 # updated. Since the rank of any change in this list can be changed any time
455 # (say user posting comment), subsequent calls may overalp like this:
456 # > initial order ABCDEFGH
457 # query[0..3] => ABC
458 # > E get's updated. New order: EABCDFGH
459 # query[3..6] => CDF # C is a dup
460 # query[6..9] => GH # E is missed.
Michael Achenbach6fbf12f2017-07-06 10:54:11 +0200461 page = QueryChanges(host, params, first_param, limit, o_params,
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100462 cur_start)
463 for cl in at_most_once(page):
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000464 yield cl
465
466 more_changes = [cl for cl in page if '_more_changes' in cl]
467 if len(more_changes) > 1:
468 raise GerritError(
469 200,
470 'Received %d changes with a _more_changes attribute set but should '
471 'receive at most one.' % len(more_changes))
472 if more_changes:
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100473 cur_start += len(page)
474
475 # If we paged through, query again the first page which in most circumstances
476 # will fetch all changes that were modified while this function was run.
477 if start != cur_start:
Michael Achenbach6fbf12f2017-07-06 10:54:11 +0200478 page = QueryChanges(host, params, first_param, limit, o_params, start)
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100479 for cl in at_most_once(page):
480 yield cl
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000481
482
Michael Achenbach6fbf12f2017-07-06 10:54:11 +0200483def MultiQueryChanges(host, params, change_list, limit=None, o_params=None,
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100484 start=None):
szager@chromium.orgb4696232013-10-16 19:45:35 +0000485 """Initiate a query composed of multiple sets of query parameters."""
486 if not change_list:
487 raise RuntimeError(
488 "MultiQueryChanges requires a list of change numbers/id's")
489 q = ['q=%s' % '+OR+'.join([urllib.quote(str(x)) for x in change_list])]
Michael Achenbach6fbf12f2017-07-06 10:54:11 +0200490 if params:
491 q.append(_QueryString(params))
szager@chromium.orgb4696232013-10-16 19:45:35 +0000492 if limit:
493 q.append('n=%d' % limit)
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100494 if start:
495 q.append('S=%s' % start)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000496 if o_params:
497 q.extend(['o=%s' % p for p in o_params])
498 path = 'changes/?%s' % '&'.join(q)
499 try:
Aaron Gable6f5a8d92017-04-18 14:49:05 -0700500 result = ReadHttpJsonResponse(CreateHttpConn(host, path))
szager@chromium.orgb4696232013-10-16 19:45:35 +0000501 except GerritError as e:
502 msg = '%s:\n%s' % (e.message, path)
503 raise GerritError(e.http_status, msg)
504 return result
505
506
507def GetGerritFetchUrl(host):
508 """Given a gerrit host name returns URL of a gerrit instance to fetch from."""
509 return '%s://%s/' % (GERRIT_PROTOCOL, host)
510
511
512def GetChangePageUrl(host, change_number):
513 """Given a gerrit host name and change number, return change page url."""
514 return '%s://%s/#/c/%d/' % (GERRIT_PROTOCOL, host, change_number)
515
516
517def GetChangeUrl(host, change):
518 """Given a gerrit host name and change id, return an url for the change."""
519 return '%s://%s/a/changes/%s' % (GERRIT_PROTOCOL, host, change)
520
521
522def GetChange(host, change):
523 """Query a gerrit server for information about a single change."""
524 path = 'changes/%s' % change
525 return ReadHttpJsonResponse(CreateHttpConn(host, path))
526
527
Aaron Gable6f5a8d92017-04-18 14:49:05 -0700528def GetChangeDetail(host, change, o_params=None):
szager@chromium.orgb4696232013-10-16 19:45:35 +0000529 """Query a gerrit server for extended information about a single change."""
530 path = 'changes/%s/detail' % change
531 if o_params:
532 path += '?%s' % '&'.join(['o=%s' % p for p in o_params])
Aaron Gable6f5a8d92017-04-18 14:49:05 -0700533 return ReadHttpJsonResponse(CreateHttpConn(host, path))
szager@chromium.orgb4696232013-10-16 19:45:35 +0000534
535
agable32978d92016-11-01 12:55:02 -0700536def GetChangeCommit(host, change, revision='current'):
537 """Query a gerrit server for a revision associated with a change."""
538 path = 'changes/%s/revisions/%s/commit?links' % (change, revision)
539 return ReadHttpJsonResponse(CreateHttpConn(host, path))
540
541
szager@chromium.orgb4696232013-10-16 19:45:35 +0000542def GetChangeCurrentRevision(host, change):
543 """Get information about the latest revision for a given change."""
Michael Achenbach6fbf12f2017-07-06 10:54:11 +0200544 return QueryChanges(host, [], change, o_params=('CURRENT_REVISION',))
szager@chromium.orgb4696232013-10-16 19:45:35 +0000545
546
547def GetChangeRevisions(host, change):
548 """Get information about all revisions associated with a change."""
Michael Achenbach6fbf12f2017-07-06 10:54:11 +0200549 return QueryChanges(host, [], change, o_params=('ALL_REVISIONS',))
szager@chromium.orgb4696232013-10-16 19:45:35 +0000550
551
552def GetChangeReview(host, change, revision=None):
553 """Get the current review information for a change."""
554 if not revision:
555 jmsg = GetChangeRevisions(host, change)
556 if not jmsg:
557 return None
558 elif len(jmsg) > 1:
559 raise GerritError(200, 'Multiple changes found for ChangeId %s.' % change)
560 revision = jmsg[0]['current_revision']
561 path = 'changes/%s/revisions/%s/review'
562 return ReadHttpJsonResponse(CreateHttpConn(host, path))
563
564
Aaron Gable0ffdf2d2017-06-05 13:01:17 -0700565def GetChangeComments(host, change):
566 """Get the line- and file-level comments on a change."""
567 path = 'changes/%s/comments' % change
568 return ReadHttpJsonResponse(CreateHttpConn(host, path))
569
570
szager@chromium.orgb4696232013-10-16 19:45:35 +0000571def AbandonChange(host, change, msg=''):
572 """Abandon a gerrit change."""
573 path = 'changes/%s/abandon' % change
tandrii@chromium.orgc7da66a2016-03-24 09:52:24 +0000574 body = {'message': msg} if msg else {}
szager@chromium.orgb4696232013-10-16 19:45:35 +0000575 conn = CreateHttpConn(host, path, reqtype='POST', body=body)
Aaron Gable6f5a8d92017-04-18 14:49:05 -0700576 return ReadHttpJsonResponse(conn)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000577
578
579def RestoreChange(host, change, msg=''):
580 """Restore a previously abandoned change."""
581 path = 'changes/%s/restore' % change
tandrii@chromium.orgc7da66a2016-03-24 09:52:24 +0000582 body = {'message': msg} if msg else {}
szager@chromium.orgb4696232013-10-16 19:45:35 +0000583 conn = CreateHttpConn(host, path, reqtype='POST', body=body)
Aaron Gable6f5a8d92017-04-18 14:49:05 -0700584 return ReadHttpJsonResponse(conn)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000585
586
587def SubmitChange(host, change, wait_for_merge=True):
588 """Submits a gerrit change via Gerrit."""
589 path = 'changes/%s/submit' % change
590 body = {'wait_for_merge': wait_for_merge}
591 conn = CreateHttpConn(host, path, reqtype='POST', body=body)
Aaron Gable6f5a8d92017-04-18 14:49:05 -0700592 return ReadHttpJsonResponse(conn)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000593
594
dsansomee2d6fd92016-09-08 00:10:47 -0700595def HasPendingChangeEdit(host, change):
596 conn = CreateHttpConn(host, 'changes/%s/edit' % change)
597 try:
Aaron Gable6f5a8d92017-04-18 14:49:05 -0700598 ReadHttpResponse(conn)
dsansomee2d6fd92016-09-08 00:10:47 -0700599 except GerritError as e:
Aaron Gable19ee16c2017-04-18 11:56:35 -0700600 # 204 No Content means no pending change.
601 if e.http_status == 204:
602 return False
603 raise
604 return True
dsansomee2d6fd92016-09-08 00:10:47 -0700605
606
607def DeletePendingChangeEdit(host, change):
608 conn = CreateHttpConn(host, 'changes/%s/edit' % change, reqtype='DELETE')
Aaron Gable19ee16c2017-04-18 11:56:35 -0700609 # On success, gerrit returns status 204; if the edit was already deleted it
610 # returns 404. Anything else is an error.
611 ReadHttpResponse(conn, accept_statuses=[204, 404])
dsansomee2d6fd92016-09-08 00:10:47 -0700612
613
Andrii Shyshkalovea4fc832016-12-01 14:53:23 +0100614def SetCommitMessage(host, change, description, notify='ALL'):
scottmg@chromium.org6d1266e2016-04-26 11:12:26 +0000615 """Updates a commit message."""
Aaron Gable7625d882017-06-26 09:47:26 -0700616 assert notify in ('ALL', 'NONE')
617 path = 'changes/%s/message' % change
618 body = {'message': description}
619 conn = CreateHttpConn(host, path, reqtype='PUT', body=body)
scottmg@chromium.org6d1266e2016-04-26 11:12:26 +0000620 try:
Aaron Gable7625d882017-06-26 09:47:26 -0700621 ReadHttpResponse(conn, accept_statuses=[200, 204])
622 except GerritError as e:
623 raise GerritError(
624 e.http_status,
625 'Received unexpected http status while editing message '
626 'in change %s' % change)
scottmg@chromium.org6d1266e2016-04-26 11:12:26 +0000627
628
szager@chromium.orgb4696232013-10-16 19:45:35 +0000629def GetReviewers(host, change):
630 """Get information about all reviewers attached to a change."""
631 path = 'changes/%s/reviewers' % change
632 return ReadHttpJsonResponse(CreateHttpConn(host, path))
633
634
635def GetReview(host, change, revision):
636 """Get review information about a specific revision of a change."""
637 path = 'changes/%s/revisions/%s/review' % (change, revision)
638 return ReadHttpJsonResponse(CreateHttpConn(host, path))
639
640
Aaron Gable6dadfbf2017-05-09 14:27:58 -0700641def AddReviewers(host, change, reviewers=None, ccs=None, notify=True,
642 accept_statuses=frozenset([200, 400, 422])):
szager@chromium.orgb4696232013-10-16 19:45:35 +0000643 """Add reviewers to a change."""
Aaron Gable6dadfbf2017-05-09 14:27:58 -0700644 if not reviewers and not ccs:
Aaron Gabledf86e302016-11-08 10:48:03 -0800645 return None
Wiktor Garbacz6d0d0442017-05-15 12:34:40 +0200646 if not change:
647 return None
Aaron Gable6dadfbf2017-05-09 14:27:58 -0700648 reviewers = frozenset(reviewers or [])
649 ccs = frozenset(ccs or [])
650 path = 'changes/%s/revisions/current/review' % change
651
652 body = {
653 'reviewers': [],
654 'notify': 'ALL' if notify else 'NONE',
655 }
656 for r in sorted(reviewers | ccs):
657 state = 'REVIEWER' if r in reviewers else 'CC'
658 body['reviewers'].append({
659 'reviewer': r,
660 'state': state,
661 'notify': 'NONE', # We handled `notify` argument above.
662 })
663
664 conn = CreateHttpConn(host, path, reqtype='POST', body=body)
665 # Gerrit will return 400 if one or more of the requested reviewers are
666 # unprocessable. We read the response object to see which were rejected,
667 # warn about them, and retry with the remainder.
668 resp = ReadHttpJsonResponse(conn, accept_statuses=accept_statuses)
669
670 errored = set()
671 for result in resp.get('reviewers', {}).itervalues():
672 r = result.get('input')
673 state = 'REVIEWER' if r in reviewers else 'CC'
674 if result.get('error'):
675 errored.add(r)
676 LOGGER.warn('Note: "%s" not added as a %s' % (r, state.lower()))
677 if errored:
678 # Try again, adding only those that didn't fail, and only accepting 200.
679 AddReviewers(host, change, reviewers=(reviewers-errored),
680 ccs=(ccs-errored), notify=notify, accept_statuses=[200])
szager@chromium.orgb4696232013-10-16 19:45:35 +0000681
682
683def RemoveReviewers(host, change, remove=None):
684 """Remove reveiewers from a change."""
685 if not remove:
686 return
687 if isinstance(remove, basestring):
688 remove = (remove,)
689 for r in remove:
690 path = 'changes/%s/reviewers/%s' % (change, r)
691 conn = CreateHttpConn(host, path, reqtype='DELETE')
692 try:
Aaron Gable19ee16c2017-04-18 11:56:35 -0700693 ReadHttpResponse(conn, accept_statuses=[204])
szager@chromium.orgb4696232013-10-16 19:45:35 +0000694 except GerritError as e:
szager@chromium.orgb4696232013-10-16 19:45:35 +0000695 raise GerritError(
Aaron Gable19ee16c2017-04-18 11:56:35 -0700696 e.http_status,
697 'Received unexpected http status while deleting reviewer "%s" '
698 'from change %s' % (r, change))
szager@chromium.orgb4696232013-10-16 19:45:35 +0000699
700
701def SetReview(host, change, msg=None, labels=None, notify=None):
702 """Set labels and/or add a message to a code review."""
703 if not msg and not labels:
704 return
705 path = 'changes/%s/revisions/current/review' % change
706 body = {}
707 if msg:
708 body['message'] = msg
709 if labels:
710 body['labels'] = labels
711 if notify:
Aaron Gable75e78722017-06-09 10:40:16 -0700712 body['notify'] = 'ALL' if notify else 'NONE'
szager@chromium.orgb4696232013-10-16 19:45:35 +0000713 conn = CreateHttpConn(host, path, reqtype='POST', body=body)
714 response = ReadHttpJsonResponse(conn)
715 if labels:
716 for key, val in labels.iteritems():
717 if ('labels' not in response or key not in response['labels'] or
718 int(response['labels'][key] != int(val))):
719 raise GerritError(200, 'Unable to set "%s" label on change %s.' % (
720 key, change))
721
722
723def ResetReviewLabels(host, change, label, value='0', message=None,
724 notify=None):
725 """Reset the value of a given label for all reviewers on a change."""
726 # This is tricky, because we want to work on the "current revision", but
727 # there's always the risk that "current revision" will change in between
728 # API calls. So, we check "current revision" at the beginning and end; if
729 # it has changed, raise an exception.
730 jmsg = GetChangeCurrentRevision(host, change)
731 if not jmsg:
732 raise GerritError(
733 200, 'Could not get review information for change "%s"' % change)
734 value = str(value)
735 revision = jmsg[0]['current_revision']
736 path = 'changes/%s/revisions/%s/review' % (change, revision)
737 message = message or (
738 '%s label set to %s programmatically.' % (label, value))
739 jmsg = GetReview(host, change, revision)
740 if not jmsg:
741 raise GerritError(200, 'Could not get review information for revison %s '
742 'of change %s' % (revision, change))
743 for review in jmsg.get('labels', {}).get(label, {}).get('all', []):
744 if str(review.get('value', value)) != value:
745 body = {
746 'message': message,
747 'labels': {label: value},
748 'on_behalf_of': review['_account_id'],
749 }
750 if notify:
751 body['notify'] = notify
752 conn = CreateHttpConn(
753 host, path, reqtype='POST', body=body)
754 response = ReadHttpJsonResponse(conn)
755 if str(response['labels'][label]) != value:
756 username = review.get('email', jmsg.get('name', ''))
757 raise GerritError(200, 'Unable to set %s label for user "%s"'
758 ' on change %s.' % (label, username, change))
759 jmsg = GetChangeCurrentRevision(host, change)
760 if not jmsg:
761 raise GerritError(
762 200, 'Could not get review information for change "%s"' % change)
763 elif jmsg[0]['current_revision'] != revision:
764 raise GerritError(200, 'While resetting labels on change "%s", '
765 'a new patchset was uploaded.' % change)
Dan Jacques8d11e482016-11-15 14:25:56 -0800766
767
dimu833c94c2017-01-18 17:36:15 -0800768def CreateGerritBranch(host, project, branch, commit):
769 """
770 Create a new branch from given project and commit
771 https://gerrit-review.googlesource.com/Documentation/rest-api-projects.html#create-branch
772
773 Returns:
774 A JSON with 'ref' key
775 """
776 path = 'projects/%s/branches/%s' % (project, branch)
777 body = {'revision': commit}
778 conn = CreateHttpConn(host, path, reqtype='PUT', body=body)
dimu7d1af2b2017-04-19 16:01:17 -0700779 response = ReadHttpJsonResponse(conn, accept_statuses=[201])
dimu833c94c2017-01-18 17:36:15 -0800780 if response:
781 return response
782 raise GerritError(200, 'Unable to create gerrit branch')
783
784
785def GetGerritBranch(host, project, branch):
786 """
787 Get a branch from given project and commit
788 https://gerrit-review.googlesource.com/Documentation/rest-api-projects.html#get-branch
789
790 Returns:
791 A JSON object with 'revision' key
792 """
793 path = 'projects/%s/branches/%s' % (project, branch)
794 conn = CreateHttpConn(host, path, reqtype='GET')
795 response = ReadHttpJsonResponse(conn)
796 if response:
797 return response
798 raise GerritError(200, 'Unable to get gerrit branch')
799
800
Andrii Shyshkalovbb86fbb2017-03-24 14:59:28 +0100801def GetAccountDetails(host, account_id='self'):
802 """Returns details of the account.
803
804 If account_id is not given, uses magic value 'self' which corresponds to
805 whichever account user is authenticating as.
806
807 Documentation:
808 https://gerrit-review.googlesource.com/Documentation/rest-api-accounts.html#get-account
809 """
810 if account_id != 'self':
811 account_id = int(account_id)
812 conn = CreateHttpConn(host, '/accounts/%s' % account_id)
813 return ReadHttpJsonResponse(conn)
814
815
Dan Jacques8d11e482016-11-15 14:25:56 -0800816@contextlib.contextmanager
817def tempdir():
818 tdir = None
819 try:
820 tdir = tempfile.mkdtemp(suffix='gerrit_util')
821 yield tdir
822 finally:
823 if tdir:
824 gclient_utils.rmtree(tdir)