blob: 1756a130040a92252ce0cfba498f6a921ff41295 [file] [log] [blame]
szager@chromium.orgb4696232013-10-16 19:45:35 +00001# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""
6Utilities for requesting information for a gerrit server via https.
7
8https://gerrit-review.googlesource.com/Documentation/rest-api.html
9"""
10
11import base64
Dan Jacques8d11e482016-11-15 14:25:56 -080012import contextlib
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +000013import cookielib
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +010014import httplib # Still used for its constants.
szager@chromium.orgb4696232013-10-16 19:45:35 +000015import json
16import logging
17import netrc
18import os
nodir@chromium.orgce32b6e2014-05-12 20:31:32 +000019import re
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +000020import socket
szager@chromium.orgf202a252014-05-27 18:55:52 +000021import stat
22import sys
Dan Jacques8d11e482016-11-15 14:25:56 -080023import tempfile
szager@chromium.orgb4696232013-10-16 19:45:35 +000024import time
25import urllib
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +000026import urlparse
szager@chromium.orgb4696232013-10-16 19:45:35 +000027from cStringIO import StringIO
28
Dan Jacques8d11e482016-11-15 14:25:56 -080029import gclient_utils
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +010030from third_party import httplib2
szager@chromium.orgf202a252014-05-27 18:55:52 +000031
szager@chromium.orgb4696232013-10-16 19:45:35 +000032LOGGER = logging.getLogger()
33TRY_LIMIT = 5
34
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +000035
szager@chromium.orgb4696232013-10-16 19:45:35 +000036# Controls the transport protocol used to communicate with gerrit.
37# This is parameterized primarily to enable GerritTestCase.
38GERRIT_PROTOCOL = 'https'
39
40
41class GerritError(Exception):
42 """Exception class for errors commuicating with the gerrit-on-borg service."""
43 def __init__(self, http_status, *args, **kwargs):
44 super(GerritError, self).__init__(*args, **kwargs)
45 self.http_status = http_status
46 self.message = '(%d) %s' % (self.http_status, self.message)
47
48
nodir@chromium.orgce32b6e2014-05-12 20:31:32 +000049class GerritAuthenticationError(GerritError):
50 """Exception class for authentication errors during Gerrit communication."""
51
52
Michael Achenbach6fbf12f2017-07-06 10:54:11 +020053def _QueryString(params, first_param=None):
szager@chromium.orgb4696232013-10-16 19:45:35 +000054 """Encodes query parameters in the key:val[+key:val...] format specified here:
55
56 https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#list-changes
57 """
58 q = [urllib.quote(first_param)] if first_param else []
Michael Achenbach6fbf12f2017-07-06 10:54:11 +020059 q.extend(['%s:%s' % (key, val) for key, val in params])
szager@chromium.orgb4696232013-10-16 19:45:35 +000060 return '+'.join(q)
61
62
Aaron Gabled2db5a22017-03-24 14:14:15 -070063def GetConnectionObject(protocol=None):
szager@chromium.orgb4696232013-10-16 19:45:35 +000064 if protocol is None:
65 protocol = GERRIT_PROTOCOL
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +010066 if protocol in ('http', 'https'):
Aaron Gabled2db5a22017-03-24 14:14:15 -070067 return httplib2.Http()
szager@chromium.orgb4696232013-10-16 19:45:35 +000068 else:
69 raise RuntimeError(
70 "Don't know how to work with protocol '%s'" % protocol)
71
72
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +000073class Authenticator(object):
74 """Base authenticator class for authenticator implementations to subclass."""
75
76 def get_auth_header(self, host):
77 raise NotImplementedError()
78
79 @staticmethod
80 def get():
81 """Returns: (Authenticator) The identified Authenticator to use.
82
83 Probes the local system and its environment and identifies the
84 Authenticator instance to use.
85 """
86 if GceAuthenticator.is_gce():
87 return GceAuthenticator()
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +000088 return CookiesAuthenticator()
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +000089
90
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +000091class CookiesAuthenticator(Authenticator):
92 """Authenticator implementation that uses ".netrc" or ".gitcookies" for token.
93
94 Expected case for developer workstations.
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +000095 """
96
97 def __init__(self):
98 self.netrc = self._get_netrc()
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +000099 self.gitcookies = self._get_gitcookies()
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000100
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000101 @classmethod
Andrii Shyshkalovc8173822017-07-10 12:10:53 +0200102 def get_new_password_url(cls, host):
103 assert not host.startswith('http')
104 # Assume *.googlesource.com pattern.
105 parts = host.split('.')
106 if not parts[0].endswith('-review'):
107 parts[0] += '-review'
108 return 'https://%s/new-password' % ('.'.join(parts))
109
110 @classmethod
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000111 def get_new_password_message(cls, host):
112 assert not host.startswith('http')
113 # Assume *.googlesource.com pattern.
114 parts = host.split('.')
115 if not parts[0].endswith('-review'):
116 parts[0] += '-review'
117 url = 'https://%s/new-password' % ('.'.join(parts))
Andrii Shyshkalov0a0b0672017-03-16 16:27:48 +0100118 return 'You can (re)generate your credentials by visiting %s' % url
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000119
120 @classmethod
121 def get_netrc_path(cls):
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000122 path = '_netrc' if sys.platform.startswith('win') else '.netrc'
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000123 return os.path.expanduser(os.path.join('~', path))
124
125 @classmethod
126 def _get_netrc(cls):
Dan Jacques8d11e482016-11-15 14:25:56 -0800127 # Buffer the '.netrc' path. Use an empty file if it doesn't exist.
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000128 path = cls.get_netrc_path()
Dan Jacques8d11e482016-11-15 14:25:56 -0800129 content = ''
130 if os.path.exists(path):
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000131 st = os.stat(path)
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000132 if st.st_mode & (stat.S_IRWXG | stat.S_IRWXO):
133 print >> sys.stderr, (
134 'WARNING: netrc file %s cannot be used because its file '
135 'permissions are insecure. netrc file permissions should be '
136 '600.' % path)
Dan Jacques8d11e482016-11-15 14:25:56 -0800137 with open(path) as fd:
138 content = fd.read()
139
140 # Load the '.netrc' file. We strip comments from it because processing them
141 # can trigger a bug in Windows. See crbug.com/664664.
142 content = '\n'.join(l for l in content.splitlines()
143 if l.strip() and not l.strip().startswith('#'))
144 with tempdir() as tdir:
145 netrc_path = os.path.join(tdir, 'netrc')
146 with open(netrc_path, 'w') as fd:
147 fd.write(content)
148 os.chmod(netrc_path, (stat.S_IRUSR | stat.S_IWUSR))
149 return cls._get_netrc_from_path(netrc_path)
150
151 @classmethod
152 def _get_netrc_from_path(cls, path):
153 try:
154 return netrc.netrc(path)
155 except IOError:
156 print >> sys.stderr, 'WARNING: Could not read netrc file %s' % path
157 return netrc.netrc(os.devnull)
158 except netrc.NetrcParseError as e:
159 print >> sys.stderr, ('ERROR: Cannot use netrc file %s due to a '
160 'parsing error: %s' % (path, e))
161 return netrc.netrc(os.devnull)
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000162
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000163 @classmethod
164 def get_gitcookies_path(cls):
Ravi Mistry0bfa9ad2016-11-21 12:58:31 -0500165 if os.getenv('GIT_COOKIES_PATH'):
166 return os.getenv('GIT_COOKIES_PATH')
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000167 return os.path.join(os.environ['HOME'], '.gitcookies')
168
169 @classmethod
170 def _get_gitcookies(cls):
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +0000171 gitcookies = {}
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000172 path = cls.get_gitcookies_path()
173 if not os.path.exists(path):
174 return gitcookies
175
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +0000176 try:
177 f = open(path, 'rb')
178 except IOError:
179 return gitcookies
180
181 with f:
182 for line in f:
183 try:
184 fields = line.strip().split('\t')
185 if line.strip().startswith('#') or len(fields) != 7:
186 continue
187 domain, xpath, key, value = fields[0], fields[2], fields[5], fields[6]
188 if xpath == '/' and key == 'o':
189 login, secret_token = value.split('=', 1)
190 gitcookies[domain] = (login, secret_token)
191 except (IndexError, ValueError, TypeError) as exc:
Andrii Shyshkalov5b04a572017-01-23 17:44:41 +0100192 LOGGER.warning(exc)
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +0000193
194 return gitcookies
195
Andrii Shyshkalovbb86fbb2017-03-24 14:59:28 +0100196 def _get_auth_for_host(self, host):
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +0000197 for domain, creds in self.gitcookies.iteritems():
198 if cookielib.domain_match(host, domain):
Andrii Shyshkalovbb86fbb2017-03-24 14:59:28 +0100199 return (creds[0], None, creds[1])
200 return self.netrc.authenticators(host)
phajdan.jr@chromium.orgff7840a2015-11-04 16:35:22 +0000201
Andrii Shyshkalovbb86fbb2017-03-24 14:59:28 +0100202 def get_auth_header(self, host):
203 auth = self._get_auth_for_host(host)
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000204 if auth:
205 return 'Basic %s' % (base64.b64encode('%s:%s' % (auth[0], auth[2])))
206 return None
207
Andrii Shyshkalovbb86fbb2017-03-24 14:59:28 +0100208 def get_auth_email(self, host):
209 """Best effort parsing of email to be used for auth for the given host."""
210 auth = self._get_auth_for_host(host)
211 if not auth:
212 return None
213 login = auth[0]
214 # login typically looks like 'git-xxx.example.com'
215 if not login.startswith('git-') or '.' not in login:
216 return None
217 username, domain = login[len('git-'):].split('.', 1)
218 return '%s@%s' % (username, domain)
219
Andrii Shyshkalov18975322017-01-25 16:44:13 +0100220
tandrii@chromium.orgfe30f182016-04-13 12:15:04 +0000221# Backwards compatibility just in case somebody imports this outside of
222# depot_tools.
223NetrcAuthenticator = CookiesAuthenticator
224
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000225
226class GceAuthenticator(Authenticator):
227 """Authenticator implementation that uses GCE metadata service for token.
228 """
229
230 _INFO_URL = 'http://metadata.google.internal'
231 _ACQUIRE_URL = ('http://metadata/computeMetadata/v1/instance/'
232 'service-accounts/default/token')
233 _ACQUIRE_HEADERS = {"Metadata-Flavor": "Google"}
234
235 _cache_is_gce = None
236 _token_cache = None
237 _token_expiration = None
238
239 @classmethod
240 def is_gce(cls):
Ravi Mistryfad941b2016-11-15 13:00:47 -0500241 if os.getenv('SKIP_GCE_AUTH_FOR_GIT'):
242 return False
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000243 if cls._cache_is_gce is None:
244 cls._cache_is_gce = cls._test_is_gce()
245 return cls._cache_is_gce
246
247 @classmethod
248 def _test_is_gce(cls):
249 # Based on https://cloud.google.com/compute/docs/metadata#runninggce
250 try:
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100251 resp, _ = cls._get(cls._INFO_URL)
252 except (socket.error, httplib2.ServerNotFoundError):
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000253 # Could not resolve URL.
254 return False
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100255 return resp.get('metadata-flavor') == 'Google'
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000256
257 @staticmethod
258 def _get(url, **kwargs):
259 next_delay_sec = 1
260 for i in xrange(TRY_LIMIT):
261 if i > 0:
262 # Retry server error status codes.
263 LOGGER.info('Encountered server error; retrying after %d second(s).',
264 next_delay_sec)
265 time.sleep(next_delay_sec)
266 next_delay_sec *= 2
267
268 p = urlparse.urlparse(url)
Aaron Gabled2db5a22017-03-24 14:14:15 -0700269 c = GetConnectionObject(protocol=p.scheme)
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100270 resp, contents = c.request(url, 'GET', **kwargs)
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000271 LOGGER.debug('GET [%s] #%d/%d (%d)', url, i+1, TRY_LIMIT, resp.status)
272 if resp.status < httplib.INTERNAL_SERVER_ERROR:
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100273 return (resp, contents)
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000274
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000275 @classmethod
276 def _get_token_dict(cls):
277 if cls._token_cache:
278 # If it expires within 25 seconds, refresh.
279 if cls._token_expiration < time.time() - 25:
280 return cls._token_cache
281
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100282 resp, contents = cls._get(cls._ACQUIRE_URL, headers=cls._ACQUIRE_HEADERS)
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000283 if resp.status != httplib.OK:
284 return None
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100285 cls._token_cache = json.loads(contents)
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000286 cls._token_expiration = cls._token_cache['expires_in'] + time.time()
287 return cls._token_cache
288
289 def get_auth_header(self, _host):
290 token_dict = self._get_token_dict()
291 if not token_dict:
292 return None
293 return '%(token_type)s %(access_token)s' % token_dict
294
295
szager@chromium.orgb4696232013-10-16 19:45:35 +0000296def CreateHttpConn(host, path, reqtype='GET', headers=None, body=None):
297 """Opens an https connection to a gerrit service, and sends a request."""
298 headers = headers or {}
299 bare_host = host.partition(':')[0]
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000300
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000301 auth = Authenticator.get().get_auth_header(bare_host)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000302 if auth:
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000303 headers.setdefault('Authorization', auth)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000304 else:
dnj@chromium.orga5a2c8a2015-09-29 16:22:55 +0000305 LOGGER.debug('No authorization found for %s.' % bare_host)
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000306
Dan Jacques6d5bcc22016-11-14 15:32:04 -0800307 url = path
308 if not url.startswith('/'):
309 url = '/' + url
310 if 'Authorization' in headers and not url.startswith('/a/'):
311 url = '/a%s' % url
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000312
szager@chromium.orgb4696232013-10-16 19:45:35 +0000313 if body:
314 body = json.JSONEncoder().encode(body)
315 headers.setdefault('Content-Type', 'application/json')
316 if LOGGER.isEnabledFor(logging.DEBUG):
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000317 LOGGER.debug('%s %s://%s%s' % (reqtype, GERRIT_PROTOCOL, host, url))
szager@chromium.orgb4696232013-10-16 19:45:35 +0000318 for key, val in headers.iteritems():
319 if key == 'Authorization':
320 val = 'HIDDEN'
321 LOGGER.debug('%s: %s' % (key, val))
322 if body:
323 LOGGER.debug(body)
Aaron Gabled2db5a22017-03-24 14:14:15 -0700324 conn = GetConnectionObject()
szager@chromium.orgb4696232013-10-16 19:45:35 +0000325 conn.req_host = host
326 conn.req_params = {
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100327 'uri': urlparse.urljoin('%s://%s' % (GERRIT_PROTOCOL, host), url),
szager@chromium.orgb4696232013-10-16 19:45:35 +0000328 'method': reqtype,
329 'headers': headers,
330 'body': body,
331 }
szager@chromium.orgb4696232013-10-16 19:45:35 +0000332 return conn
333
334
Aaron Gable6f5a8d92017-04-18 14:49:05 -0700335def ReadHttpResponse(conn, accept_statuses=frozenset([200])):
szager@chromium.orgb4696232013-10-16 19:45:35 +0000336 """Reads an http response from a connection into a string buffer.
337
338 Args:
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100339 conn: An Http object created by CreateHttpConn above.
Aaron Gable6f5a8d92017-04-18 14:49:05 -0700340 accept_statuses: Treat any of these statuses as success. Default: [200]
341 Common additions include 204, 400, and 404.
szager@chromium.orgb4696232013-10-16 19:45:35 +0000342 Returns: A string buffer containing the connection's reply.
343 """
szager@chromium.orgb4696232013-10-16 19:45:35 +0000344 sleep_time = 0.5
345 for idx in range(TRY_LIMIT):
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100346 response, contents = conn.request(**conn.req_params)
nodir@chromium.orgce32b6e2014-05-12 20:31:32 +0000347
348 # Check if this is an authentication issue.
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100349 www_authenticate = response.get('www-authenticate')
nodir@chromium.orgce32b6e2014-05-12 20:31:32 +0000350 if (response.status in (httplib.UNAUTHORIZED, httplib.FOUND) and
351 www_authenticate):
352 auth_match = re.search('realm="([^"]+)"', www_authenticate, re.I)
353 host = auth_match.group(1) if auth_match else conn.req_host
Aaron Gable19ee16c2017-04-18 11:56:35 -0700354 reason = ('Authentication failed. Please make sure your .gitcookies file '
nodir@chromium.orgce32b6e2014-05-12 20:31:32 +0000355 'has credentials for %s' % host)
356 raise GerritAuthenticationError(response.status, reason)
357
szager@chromium.orgb4696232013-10-16 19:45:35 +0000358 # If response.status < 500 then the result is final; break retry loop.
Aaron Gable62ca9602017-05-19 17:24:52 -0700359 # If the response is 404, it might be because of replication lag, so
360 # keep trying anyway.
361 if response.status < 500 and response.status != 404:
Andrii Shyshkalov5b04a572017-01-23 17:44:41 +0100362 LOGGER.debug('got response %d for %s %s', response.status,
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100363 conn.req_params['method'], conn.req_params['uri'])
szager@chromium.orgb4696232013-10-16 19:45:35 +0000364 break
365 # A status >=500 is assumed to be a possible transient error; retry.
366 http_version = 'HTTP/%s' % ('1.1' if response.version == 11 else '1.0')
Andrii Shyshkalov5b04a572017-01-23 17:44:41 +0100367 LOGGER.warn('A transient error occurred while querying %s:\n'
368 '%s %s %s\n'
369 '%s %d %s',
Aaron Gable20d2cbb2017-04-25 15:04:05 -0700370 conn.req_host, conn.req_params['method'],
371 conn.req_params['uri'],
Andrii Shyshkalov5b04a572017-01-23 17:44:41 +0100372 http_version, http_version, response.status, response.reason)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000373 if TRY_LIMIT - idx > 1:
Andrii Shyshkalov5b04a572017-01-23 17:44:41 +0100374 LOGGER.warn('... will retry %d more times.', TRY_LIMIT - idx - 1)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000375 time.sleep(sleep_time)
376 sleep_time = sleep_time * 2
Aaron Gable19ee16c2017-04-18 11:56:35 -0700377 if response.status not in accept_statuses:
Andrii Shyshkalov4956f792017-04-10 14:28:38 +0200378 if response.status in (401, 403):
379 print('Your Gerrit credentials might be misconfigured. Try: \n'
380 ' git cl creds-check')
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100381 reason = '%s: %s' % (response.reason, contents)
nodir@chromium.orga7798032014-04-30 23:40:53 +0000382 raise GerritError(response.status, reason)
Raphael Kubo da Costa89d04852017-03-23 19:04:31 +0100383 return StringIO(contents)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000384
385
Aaron Gable6f5a8d92017-04-18 14:49:05 -0700386def ReadHttpJsonResponse(conn, accept_statuses=frozenset([200])):
szager@chromium.orgb4696232013-10-16 19:45:35 +0000387 """Parses an https response as json."""
Aaron Gable19ee16c2017-04-18 11:56:35 -0700388 fh = ReadHttpResponse(conn, accept_statuses)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000389 # The first line of the response should always be: )]}'
390 s = fh.readline()
391 if s and s.rstrip() != ")]}'":
392 raise GerritError(200, 'Unexpected json output: %s' % s)
393 s = fh.read()
394 if not s:
395 return None
396 return json.loads(s)
397
398
Michael Achenbach6fbf12f2017-07-06 10:54:11 +0200399def QueryChanges(host, params, first_param=None, limit=None, o_params=None,
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100400 start=None):
szager@chromium.orgb4696232013-10-16 19:45:35 +0000401 """
402 Queries a gerrit-on-borg server for changes matching query terms.
403
404 Args:
Michael Achenbach6fbf12f2017-07-06 10:54:11 +0200405 params: A list of key:value pairs for search parameters, as documented
406 here (e.g. ('is', 'owner') for a parameter 'is:owner'):
407 https://gerrit-review.googlesource.com/Documentation/user-search.html#search-operators
szager@chromium.orgb4696232013-10-16 19:45:35 +0000408 first_param: A change identifier
409 limit: Maximum number of results to return.
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100410 start: how many changes to skip (starting with the most recent)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000411 o_params: A list of additional output specifiers, as documented here:
412 https://gerrit-review.googlesource.com/Documentation/rest-api-changes.html#list-changes
413 Returns:
414 A list of json-decoded query results.
415 """
416 # Note that no attempt is made to escape special characters; YMMV.
Michael Achenbach6fbf12f2017-07-06 10:54:11 +0200417 if not params and not first_param:
szager@chromium.orgb4696232013-10-16 19:45:35 +0000418 raise RuntimeError('QueryChanges requires search parameters')
Michael Achenbach6fbf12f2017-07-06 10:54:11 +0200419 path = 'changes/?q=%s' % _QueryString(params, first_param)
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100420 if start:
421 path = '%s&start=%s' % (path, start)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000422 if limit:
423 path = '%s&n=%d' % (path, limit)
424 if o_params:
425 path = '%s&%s' % (path, '&'.join(['o=%s' % p for p in o_params]))
Aaron Gable6f5a8d92017-04-18 14:49:05 -0700426 return ReadHttpJsonResponse(CreateHttpConn(host, path))
szager@chromium.orgb4696232013-10-16 19:45:35 +0000427
428
Michael Achenbach6fbf12f2017-07-06 10:54:11 +0200429def GenerateAllChanges(host, params, first_param=None, limit=500,
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100430 o_params=None, start=None):
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000431 """
432 Queries a gerrit-on-borg server for all the changes matching the query terms.
433
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100434 WARNING: this is unreliable if a change matching the query is modified while
435 this function is being called.
436
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000437 A single query to gerrit-on-borg is limited on the number of results by the
438 limit parameter on the request (see QueryChanges) and the server maximum
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100439 limit.
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000440
441 Args:
Michael Achenbach6fbf12f2017-07-06 10:54:11 +0200442 params, first_param: Refer to QueryChanges().
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000443 limit: Maximum number of requested changes per query.
444 o_params: Refer to QueryChanges().
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100445 start: Refer to QueryChanges().
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000446
447 Returns:
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100448 A generator object to the list of returned changes.
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000449 """
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100450 already_returned = set()
451 def at_most_once(cls):
452 for cl in cls:
453 if cl['_number'] not in already_returned:
454 already_returned.add(cl['_number'])
455 yield cl
456
457 start = start or 0
458 cur_start = start
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000459 more_changes = True
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100460
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000461 while more_changes:
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100462 # This will fetch changes[start..start+limit] sorted by most recently
463 # updated. Since the rank of any change in this list can be changed any time
464 # (say user posting comment), subsequent calls may overalp like this:
465 # > initial order ABCDEFGH
466 # query[0..3] => ABC
467 # > E get's updated. New order: EABCDFGH
468 # query[3..6] => CDF # C is a dup
469 # query[6..9] => GH # E is missed.
Michael Achenbach6fbf12f2017-07-06 10:54:11 +0200470 page = QueryChanges(host, params, first_param, limit, o_params,
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100471 cur_start)
472 for cl in at_most_once(page):
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000473 yield cl
474
475 more_changes = [cl for cl in page if '_more_changes' in cl]
476 if len(more_changes) > 1:
477 raise GerritError(
478 200,
479 'Received %d changes with a _more_changes attribute set but should '
480 'receive at most one.' % len(more_changes))
481 if more_changes:
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100482 cur_start += len(page)
483
484 # If we paged through, query again the first page which in most circumstances
485 # will fetch all changes that were modified while this function was run.
486 if start != cur_start:
Michael Achenbach6fbf12f2017-07-06 10:54:11 +0200487 page = QueryChanges(host, params, first_param, limit, o_params, start)
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100488 for cl in at_most_once(page):
489 yield cl
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000490
491
Michael Achenbach6fbf12f2017-07-06 10:54:11 +0200492def MultiQueryChanges(host, params, change_list, limit=None, o_params=None,
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100493 start=None):
szager@chromium.orgb4696232013-10-16 19:45:35 +0000494 """Initiate a query composed of multiple sets of query parameters."""
495 if not change_list:
496 raise RuntimeError(
497 "MultiQueryChanges requires a list of change numbers/id's")
498 q = ['q=%s' % '+OR+'.join([urllib.quote(str(x)) for x in change_list])]
Michael Achenbach6fbf12f2017-07-06 10:54:11 +0200499 if params:
500 q.append(_QueryString(params))
szager@chromium.orgb4696232013-10-16 19:45:35 +0000501 if limit:
502 q.append('n=%d' % limit)
Andrii Shyshkalov892e9c22017-03-08 16:21:21 +0100503 if start:
504 q.append('S=%s' % start)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000505 if o_params:
506 q.extend(['o=%s' % p for p in o_params])
507 path = 'changes/?%s' % '&'.join(q)
508 try:
Aaron Gable6f5a8d92017-04-18 14:49:05 -0700509 result = ReadHttpJsonResponse(CreateHttpConn(host, path))
szager@chromium.orgb4696232013-10-16 19:45:35 +0000510 except GerritError as e:
511 msg = '%s:\n%s' % (e.message, path)
512 raise GerritError(e.http_status, msg)
513 return result
514
515
516def GetGerritFetchUrl(host):
517 """Given a gerrit host name returns URL of a gerrit instance to fetch from."""
518 return '%s://%s/' % (GERRIT_PROTOCOL, host)
519
520
521def GetChangePageUrl(host, change_number):
522 """Given a gerrit host name and change number, return change page url."""
523 return '%s://%s/#/c/%d/' % (GERRIT_PROTOCOL, host, change_number)
524
525
526def GetChangeUrl(host, change):
527 """Given a gerrit host name and change id, return an url for the change."""
528 return '%s://%s/a/changes/%s' % (GERRIT_PROTOCOL, host, change)
529
530
531def GetChange(host, change):
532 """Query a gerrit server for information about a single change."""
533 path = 'changes/%s' % change
534 return ReadHttpJsonResponse(CreateHttpConn(host, path))
535
536
Aaron Gable6f5a8d92017-04-18 14:49:05 -0700537def GetChangeDetail(host, change, o_params=None):
szager@chromium.orgb4696232013-10-16 19:45:35 +0000538 """Query a gerrit server for extended information about a single change."""
539 path = 'changes/%s/detail' % change
540 if o_params:
541 path += '?%s' % '&'.join(['o=%s' % p for p in o_params])
Aaron Gable6f5a8d92017-04-18 14:49:05 -0700542 return ReadHttpJsonResponse(CreateHttpConn(host, path))
szager@chromium.orgb4696232013-10-16 19:45:35 +0000543
544
agable32978d92016-11-01 12:55:02 -0700545def GetChangeCommit(host, change, revision='current'):
546 """Query a gerrit server for a revision associated with a change."""
547 path = 'changes/%s/revisions/%s/commit?links' % (change, revision)
548 return ReadHttpJsonResponse(CreateHttpConn(host, path))
549
550
szager@chromium.orgb4696232013-10-16 19:45:35 +0000551def GetChangeCurrentRevision(host, change):
552 """Get information about the latest revision for a given change."""
Michael Achenbach6fbf12f2017-07-06 10:54:11 +0200553 return QueryChanges(host, [], change, o_params=('CURRENT_REVISION',))
szager@chromium.orgb4696232013-10-16 19:45:35 +0000554
555
556def GetChangeRevisions(host, change):
557 """Get information about all revisions associated with a change."""
Michael Achenbach6fbf12f2017-07-06 10:54:11 +0200558 return QueryChanges(host, [], change, o_params=('ALL_REVISIONS',))
szager@chromium.orgb4696232013-10-16 19:45:35 +0000559
560
561def GetChangeReview(host, change, revision=None):
562 """Get the current review information for a change."""
563 if not revision:
564 jmsg = GetChangeRevisions(host, change)
565 if not jmsg:
566 return None
567 elif len(jmsg) > 1:
568 raise GerritError(200, 'Multiple changes found for ChangeId %s.' % change)
569 revision = jmsg[0]['current_revision']
570 path = 'changes/%s/revisions/%s/review'
571 return ReadHttpJsonResponse(CreateHttpConn(host, path))
572
573
Aaron Gable0ffdf2d2017-06-05 13:01:17 -0700574def GetChangeComments(host, change):
575 """Get the line- and file-level comments on a change."""
576 path = 'changes/%s/comments' % change
577 return ReadHttpJsonResponse(CreateHttpConn(host, path))
578
579
szager@chromium.orgb4696232013-10-16 19:45:35 +0000580def AbandonChange(host, change, msg=''):
581 """Abandon a gerrit change."""
582 path = 'changes/%s/abandon' % change
tandrii@chromium.orgc7da66a2016-03-24 09:52:24 +0000583 body = {'message': msg} if msg else {}
szager@chromium.orgb4696232013-10-16 19:45:35 +0000584 conn = CreateHttpConn(host, path, reqtype='POST', body=body)
Aaron Gable6f5a8d92017-04-18 14:49:05 -0700585 return ReadHttpJsonResponse(conn)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000586
587
588def RestoreChange(host, change, msg=''):
589 """Restore a previously abandoned change."""
590 path = 'changes/%s/restore' % change
tandrii@chromium.orgc7da66a2016-03-24 09:52:24 +0000591 body = {'message': msg} if msg else {}
szager@chromium.orgb4696232013-10-16 19:45:35 +0000592 conn = CreateHttpConn(host, path, reqtype='POST', body=body)
Aaron Gable6f5a8d92017-04-18 14:49:05 -0700593 return ReadHttpJsonResponse(conn)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000594
595
596def SubmitChange(host, change, wait_for_merge=True):
597 """Submits a gerrit change via Gerrit."""
598 path = 'changes/%s/submit' % change
599 body = {'wait_for_merge': wait_for_merge}
600 conn = CreateHttpConn(host, path, reqtype='POST', body=body)
Aaron Gable6f5a8d92017-04-18 14:49:05 -0700601 return ReadHttpJsonResponse(conn)
szager@chromium.orgb4696232013-10-16 19:45:35 +0000602
603
dsansomee2d6fd92016-09-08 00:10:47 -0700604def HasPendingChangeEdit(host, change):
605 conn = CreateHttpConn(host, 'changes/%s/edit' % change)
606 try:
Aaron Gable6f5a8d92017-04-18 14:49:05 -0700607 ReadHttpResponse(conn)
dsansomee2d6fd92016-09-08 00:10:47 -0700608 except GerritError as e:
Aaron Gable19ee16c2017-04-18 11:56:35 -0700609 # 204 No Content means no pending change.
610 if e.http_status == 204:
611 return False
612 raise
613 return True
dsansomee2d6fd92016-09-08 00:10:47 -0700614
615
616def DeletePendingChangeEdit(host, change):
617 conn = CreateHttpConn(host, 'changes/%s/edit' % change, reqtype='DELETE')
Aaron Gable19ee16c2017-04-18 11:56:35 -0700618 # On success, gerrit returns status 204; if the edit was already deleted it
619 # returns 404. Anything else is an error.
620 ReadHttpResponse(conn, accept_statuses=[204, 404])
dsansomee2d6fd92016-09-08 00:10:47 -0700621
622
Andrii Shyshkalovea4fc832016-12-01 14:53:23 +0100623def SetCommitMessage(host, change, description, notify='ALL'):
scottmg@chromium.org6d1266e2016-04-26 11:12:26 +0000624 """Updates a commit message."""
Aaron Gable7625d882017-06-26 09:47:26 -0700625 assert notify in ('ALL', 'NONE')
626 path = 'changes/%s/message' % change
627 body = {'message': description}
628 conn = CreateHttpConn(host, path, reqtype='PUT', body=body)
scottmg@chromium.org6d1266e2016-04-26 11:12:26 +0000629 try:
Aaron Gable7625d882017-06-26 09:47:26 -0700630 ReadHttpResponse(conn, accept_statuses=[200, 204])
631 except GerritError as e:
632 raise GerritError(
633 e.http_status,
634 'Received unexpected http status while editing message '
635 'in change %s' % change)
scottmg@chromium.org6d1266e2016-04-26 11:12:26 +0000636
637
szager@chromium.orgb4696232013-10-16 19:45:35 +0000638def GetReviewers(host, change):
639 """Get information about all reviewers attached to a change."""
640 path = 'changes/%s/reviewers' % change
641 return ReadHttpJsonResponse(CreateHttpConn(host, path))
642
643
644def GetReview(host, change, revision):
645 """Get review information about a specific revision of a change."""
646 path = 'changes/%s/revisions/%s/review' % (change, revision)
647 return ReadHttpJsonResponse(CreateHttpConn(host, path))
648
649
Aaron Gable6dadfbf2017-05-09 14:27:58 -0700650def AddReviewers(host, change, reviewers=None, ccs=None, notify=True,
651 accept_statuses=frozenset([200, 400, 422])):
szager@chromium.orgb4696232013-10-16 19:45:35 +0000652 """Add reviewers to a change."""
Aaron Gable6dadfbf2017-05-09 14:27:58 -0700653 if not reviewers and not ccs:
Aaron Gabledf86e302016-11-08 10:48:03 -0800654 return None
Wiktor Garbacz6d0d0442017-05-15 12:34:40 +0200655 if not change:
656 return None
Aaron Gable6dadfbf2017-05-09 14:27:58 -0700657 reviewers = frozenset(reviewers or [])
658 ccs = frozenset(ccs or [])
659 path = 'changes/%s/revisions/current/review' % change
660
661 body = {
662 'reviewers': [],
663 'notify': 'ALL' if notify else 'NONE',
664 }
665 for r in sorted(reviewers | ccs):
666 state = 'REVIEWER' if r in reviewers else 'CC'
667 body['reviewers'].append({
668 'reviewer': r,
669 'state': state,
670 'notify': 'NONE', # We handled `notify` argument above.
671 })
672
673 conn = CreateHttpConn(host, path, reqtype='POST', body=body)
674 # Gerrit will return 400 if one or more of the requested reviewers are
675 # unprocessable. We read the response object to see which were rejected,
676 # warn about them, and retry with the remainder.
677 resp = ReadHttpJsonResponse(conn, accept_statuses=accept_statuses)
678
679 errored = set()
680 for result in resp.get('reviewers', {}).itervalues():
681 r = result.get('input')
682 state = 'REVIEWER' if r in reviewers else 'CC'
683 if result.get('error'):
684 errored.add(r)
685 LOGGER.warn('Note: "%s" not added as a %s' % (r, state.lower()))
686 if errored:
687 # Try again, adding only those that didn't fail, and only accepting 200.
688 AddReviewers(host, change, reviewers=(reviewers-errored),
689 ccs=(ccs-errored), notify=notify, accept_statuses=[200])
szager@chromium.orgb4696232013-10-16 19:45:35 +0000690
691
692def RemoveReviewers(host, change, remove=None):
693 """Remove reveiewers from a change."""
694 if not remove:
695 return
696 if isinstance(remove, basestring):
697 remove = (remove,)
698 for r in remove:
699 path = 'changes/%s/reviewers/%s' % (change, r)
700 conn = CreateHttpConn(host, path, reqtype='DELETE')
701 try:
Aaron Gable19ee16c2017-04-18 11:56:35 -0700702 ReadHttpResponse(conn, accept_statuses=[204])
szager@chromium.orgb4696232013-10-16 19:45:35 +0000703 except GerritError as e:
szager@chromium.orgb4696232013-10-16 19:45:35 +0000704 raise GerritError(
Aaron Gable19ee16c2017-04-18 11:56:35 -0700705 e.http_status,
706 'Received unexpected http status while deleting reviewer "%s" '
707 'from change %s' % (r, change))
szager@chromium.orgb4696232013-10-16 19:45:35 +0000708
709
Aaron Gable636b13f2017-07-14 10:42:48 -0700710def SetReview(host, change, msg=None, labels=None, notify=None, ready=None):
szager@chromium.orgb4696232013-10-16 19:45:35 +0000711 """Set labels and/or add a message to a code review."""
712 if not msg and not labels:
713 return
714 path = 'changes/%s/revisions/current/review' % change
715 body = {}
716 if msg:
717 body['message'] = msg
718 if labels:
719 body['labels'] = labels
720 if notify:
Aaron Gable75e78722017-06-09 10:40:16 -0700721 body['notify'] = 'ALL' if notify else 'NONE'
Aaron Gable636b13f2017-07-14 10:42:48 -0700722 if ready:
723 body['ready'] = True
szager@chromium.orgb4696232013-10-16 19:45:35 +0000724 conn = CreateHttpConn(host, path, reqtype='POST', body=body)
725 response = ReadHttpJsonResponse(conn)
726 if labels:
727 for key, val in labels.iteritems():
728 if ('labels' not in response or key not in response['labels'] or
729 int(response['labels'][key] != int(val))):
730 raise GerritError(200, 'Unable to set "%s" label on change %s.' % (
731 key, change))
732
733
734def ResetReviewLabels(host, change, label, value='0', message=None,
735 notify=None):
736 """Reset the value of a given label for all reviewers on a change."""
737 # This is tricky, because we want to work on the "current revision", but
738 # there's always the risk that "current revision" will change in between
739 # API calls. So, we check "current revision" at the beginning and end; if
740 # it has changed, raise an exception.
741 jmsg = GetChangeCurrentRevision(host, change)
742 if not jmsg:
743 raise GerritError(
744 200, 'Could not get review information for change "%s"' % change)
745 value = str(value)
746 revision = jmsg[0]['current_revision']
747 path = 'changes/%s/revisions/%s/review' % (change, revision)
748 message = message or (
749 '%s label set to %s programmatically.' % (label, value))
750 jmsg = GetReview(host, change, revision)
751 if not jmsg:
752 raise GerritError(200, 'Could not get review information for revison %s '
753 'of change %s' % (revision, change))
754 for review in jmsg.get('labels', {}).get(label, {}).get('all', []):
755 if str(review.get('value', value)) != value:
756 body = {
757 'message': message,
758 'labels': {label: value},
759 'on_behalf_of': review['_account_id'],
760 }
761 if notify:
762 body['notify'] = notify
763 conn = CreateHttpConn(
764 host, path, reqtype='POST', body=body)
765 response = ReadHttpJsonResponse(conn)
766 if str(response['labels'][label]) != value:
767 username = review.get('email', jmsg.get('name', ''))
768 raise GerritError(200, 'Unable to set %s label for user "%s"'
769 ' on change %s.' % (label, username, change))
770 jmsg = GetChangeCurrentRevision(host, change)
771 if not jmsg:
772 raise GerritError(
773 200, 'Could not get review information for change "%s"' % change)
774 elif jmsg[0]['current_revision'] != revision:
775 raise GerritError(200, 'While resetting labels on change "%s", '
776 'a new patchset was uploaded.' % change)
Dan Jacques8d11e482016-11-15 14:25:56 -0800777
778
dimu833c94c2017-01-18 17:36:15 -0800779def CreateGerritBranch(host, project, branch, commit):
780 """
781 Create a new branch from given project and commit
782 https://gerrit-review.googlesource.com/Documentation/rest-api-projects.html#create-branch
783
784 Returns:
785 A JSON with 'ref' key
786 """
787 path = 'projects/%s/branches/%s' % (project, branch)
788 body = {'revision': commit}
789 conn = CreateHttpConn(host, path, reqtype='PUT', body=body)
dimu7d1af2b2017-04-19 16:01:17 -0700790 response = ReadHttpJsonResponse(conn, accept_statuses=[201])
dimu833c94c2017-01-18 17:36:15 -0800791 if response:
792 return response
793 raise GerritError(200, 'Unable to create gerrit branch')
794
795
796def GetGerritBranch(host, project, branch):
797 """
798 Get a branch from given project and commit
799 https://gerrit-review.googlesource.com/Documentation/rest-api-projects.html#get-branch
800
801 Returns:
802 A JSON object with 'revision' key
803 """
804 path = 'projects/%s/branches/%s' % (project, branch)
805 conn = CreateHttpConn(host, path, reqtype='GET')
806 response = ReadHttpJsonResponse(conn)
807 if response:
808 return response
809 raise GerritError(200, 'Unable to get gerrit branch')
810
811
Andrii Shyshkalovbb86fbb2017-03-24 14:59:28 +0100812def GetAccountDetails(host, account_id='self'):
813 """Returns details of the account.
814
815 If account_id is not given, uses magic value 'self' which corresponds to
816 whichever account user is authenticating as.
817
818 Documentation:
819 https://gerrit-review.googlesource.com/Documentation/rest-api-accounts.html#get-account
820 """
821 if account_id != 'self':
822 account_id = int(account_id)
823 conn = CreateHttpConn(host, '/accounts/%s' % account_id)
824 return ReadHttpJsonResponse(conn)
825
826
Dan Jacques8d11e482016-11-15 14:25:56 -0800827@contextlib.contextmanager
828def tempdir():
829 tdir = None
830 try:
831 tdir = tempfile.mkdtemp(suffix='gerrit_util')
832 yield tdir
833 finally:
834 if tdir:
835 gclient_utils.rmtree(tdir)