cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 1 | #!/usr/bin/env python |
| 2 | # Copyright (c) 2012 The Chromium Authors. All rights reserved. |
| 3 | # Use of this source code is governed by a BSD-style license that can be |
| 4 | # found in the LICENSE file. |
| 5 | |
| 6 | """Get stats about your activity. |
| 7 | |
| 8 | Example: |
| 9 | - my_activity.py for stats for the current week (last week on mondays). |
| 10 | - my_activity.py -Q for stats for last quarter. |
| 11 | - my_activity.py -Y for stats for this year. |
| 12 | - my_activity.py -b 4/5/12 for stats since 4/5/12. |
| 13 | - my_activity.py -b 4/5/12 -e 6/7/12 for stats between 4/5/12 and 6/7/12. |
| 14 | """ |
| 15 | |
| 16 | # These services typically only provide a created time and a last modified time |
| 17 | # for each item for general queries. This is not enough to determine if there |
| 18 | # was activity in a given time period. So, we first query for all things created |
| 19 | # before end and modified after begin. Then, we get the details of each item and |
| 20 | # check those details to determine if there was activity in the given period. |
| 21 | # This means that query time scales mostly with (today() - begin). |
| 22 | |
| 23 | import cookielib |
| 24 | import datetime |
| 25 | from datetime import datetime |
| 26 | from datetime import timedelta |
| 27 | from functools import partial |
| 28 | import json |
| 29 | import optparse |
| 30 | import os |
| 31 | import subprocess |
| 32 | import sys |
| 33 | import urllib |
| 34 | import urllib2 |
| 35 | |
deymo@chromium.org | f8be276 | 2013-11-06 01:01:59 +0000 | [diff] [blame] | 36 | import gerrit_util |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 37 | import rietveld |
| 38 | from third_party import upload |
| 39 | |
| 40 | try: |
| 41 | from dateutil.relativedelta import relativedelta # pylint: disable=F0401 |
| 42 | except ImportError: |
| 43 | print 'python-dateutil package required' |
| 44 | exit(1) |
| 45 | |
| 46 | # python-keyring provides easy access to the system keyring. |
| 47 | try: |
| 48 | import keyring # pylint: disable=W0611,F0401 |
| 49 | except ImportError: |
| 50 | print 'Consider installing python-keyring' |
| 51 | |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 52 | rietveld_instances = [ |
| 53 | { |
| 54 | 'url': 'codereview.chromium.org', |
| 55 | 'shorturl': 'crrev.com', |
| 56 | 'supports_owner_modified_query': True, |
| 57 | 'requires_auth': False, |
| 58 | 'email_domain': 'chromium.org', |
| 59 | }, |
| 60 | { |
| 61 | 'url': 'chromereviews.googleplex.com', |
| 62 | 'shorturl': 'go/chromerev', |
| 63 | 'supports_owner_modified_query': True, |
| 64 | 'requires_auth': True, |
| 65 | 'email_domain': 'google.com', |
| 66 | }, |
| 67 | { |
| 68 | 'url': 'codereview.appspot.com', |
| 69 | 'supports_owner_modified_query': True, |
| 70 | 'requires_auth': False, |
| 71 | 'email_domain': 'chromium.org', |
| 72 | }, |
| 73 | { |
| 74 | 'url': 'breakpad.appspot.com', |
| 75 | 'supports_owner_modified_query': False, |
| 76 | 'requires_auth': False, |
| 77 | 'email_domain': 'chromium.org', |
| 78 | }, |
| 79 | ] |
| 80 | |
| 81 | gerrit_instances = [ |
| 82 | { |
deymo@chromium.org | 6c03920 | 2013-09-12 12:28:12 +0000 | [diff] [blame] | 83 | 'url': 'chromium-review.googlesource.com', |
deymo@chromium.org | e52bd5a | 2013-08-29 18:05:21 +0000 | [diff] [blame] | 84 | 'shorturl': 'crosreview.com', |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 85 | }, |
deymo@chromium.org | f8be276 | 2013-11-06 01:01:59 +0000 | [diff] [blame] | 86 | { |
| 87 | 'url': 'chrome-internal-review.googlesource.com', |
| 88 | 'shorturl': 'crosreview.com/i', |
| 89 | }, |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 90 | { |
deymo@chromium.org | 6c03920 | 2013-09-12 12:28:12 +0000 | [diff] [blame] | 91 | 'host': 'gerrit.chromium.org', |
| 92 | 'port': 29418, |
| 93 | }, |
| 94 | { |
| 95 | 'host': 'gerrit-int.chromium.org', |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 96 | 'port': 29419, |
| 97 | }, |
| 98 | ] |
| 99 | |
| 100 | google_code_projects = [ |
| 101 | { |
| 102 | 'name': 'chromium', |
| 103 | 'shorturl': 'crbug.com', |
| 104 | }, |
| 105 | { |
| 106 | 'name': 'chromium-os', |
deymo@chromium.org | c840e21 | 2013-02-13 20:40:22 +0000 | [diff] [blame] | 107 | 'shorturl': 'crosbug.com', |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 108 | }, |
| 109 | { |
| 110 | 'name': 'chrome-os-partner', |
| 111 | }, |
| 112 | { |
| 113 | 'name': 'google-breakpad', |
| 114 | }, |
| 115 | { |
| 116 | 'name': 'gyp', |
enne@chromium.org | f01fad3 | 2012-11-26 18:09:38 +0000 | [diff] [blame] | 117 | }, |
| 118 | { |
| 119 | 'name': 'skia', |
| 120 | }, |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 121 | ] |
| 122 | |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 123 | # Uses ClientLogin to authenticate the user for Google Code issue trackers. |
| 124 | def get_auth_token(email): |
cjhopman@chromium.org | 3365f2d | 2012-11-01 18:53:13 +0000 | [diff] [blame] | 125 | # KeyringCreds will use the system keyring on the first try, and prompt for |
| 126 | # a password on the next ones. |
| 127 | creds = upload.KeyringCreds('code.google.com', 'code.google.com', email) |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 128 | for _ in xrange(3): |
cjhopman@chromium.org | 3365f2d | 2012-11-01 18:53:13 +0000 | [diff] [blame] | 129 | email, password = creds.GetUserCredentials() |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 130 | url = 'https://www.google.com/accounts/ClientLogin' |
| 131 | data = urllib.urlencode({ |
| 132 | 'Email': email, |
| 133 | 'Passwd': password, |
| 134 | 'service': 'code', |
| 135 | 'source': 'chrome-my-activity', |
| 136 | 'accountType': 'GOOGLE', |
| 137 | }) |
| 138 | req = urllib2.Request(url, data=data, headers={'Accept': 'text/plain'}) |
| 139 | try: |
| 140 | response = urllib2.urlopen(req) |
| 141 | response_body = response.read() |
| 142 | response_dict = dict(x.split('=') |
| 143 | for x in response_body.split('\n') if x) |
| 144 | return response_dict['Auth'] |
| 145 | except urllib2.HTTPError, e: |
cjhopman@chromium.org | 3365f2d | 2012-11-01 18:53:13 +0000 | [diff] [blame] | 146 | print e |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 147 | |
cjhopman@chromium.org | 3365f2d | 2012-11-01 18:53:13 +0000 | [diff] [blame] | 148 | print 'Unable to authenticate to code.google.com.' |
| 149 | print 'Some issues may be missing.' |
| 150 | return None |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 151 | |
| 152 | |
| 153 | def username(email): |
| 154 | """Keeps the username of an email address.""" |
| 155 | return email and email.split('@', 1)[0] |
| 156 | |
| 157 | |
cjhopman@chromium.org | 426557a | 2012-10-22 20:18:52 +0000 | [diff] [blame] | 158 | def datetime_to_midnight(date): |
| 159 | return date - timedelta(hours=date.hour, minutes=date.minute, |
| 160 | seconds=date.second, microseconds=date.microsecond) |
| 161 | |
| 162 | |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 163 | def get_quarter_of(date): |
cjhopman@chromium.org | 426557a | 2012-10-22 20:18:52 +0000 | [diff] [blame] | 164 | begin = (datetime_to_midnight(date) - |
| 165 | relativedelta(months=(date.month % 3) - 1, days=(date.day - 1))) |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 166 | return begin, begin + relativedelta(months=3) |
| 167 | |
| 168 | |
| 169 | def get_year_of(date): |
cjhopman@chromium.org | 426557a | 2012-10-22 20:18:52 +0000 | [diff] [blame] | 170 | begin = (datetime_to_midnight(date) - |
| 171 | relativedelta(months=(date.month - 1), days=(date.day - 1))) |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 172 | return begin, begin + relativedelta(years=1) |
| 173 | |
| 174 | |
| 175 | def get_week_of(date): |
cjhopman@chromium.org | 426557a | 2012-10-22 20:18:52 +0000 | [diff] [blame] | 176 | begin = (datetime_to_midnight(date) - timedelta(days=date.weekday())) |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 177 | return begin, begin + timedelta(days=7) |
| 178 | |
| 179 | |
| 180 | def get_yes_or_no(msg): |
| 181 | while True: |
| 182 | response = raw_input(msg + ' yes/no [no] ') |
| 183 | if response == 'y' or response == 'yes': |
| 184 | return True |
| 185 | elif not response or response == 'n' or response == 'no': |
| 186 | return False |
| 187 | |
| 188 | |
deymo@chromium.org | 6c03920 | 2013-09-12 12:28:12 +0000 | [diff] [blame] | 189 | def datetime_from_gerrit(date_string): |
| 190 | return datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S.%f000') |
| 191 | |
| 192 | |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 193 | def datetime_from_rietveld(date_string): |
| 194 | return datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S.%f') |
| 195 | |
| 196 | |
| 197 | def datetime_from_google_code(date_string): |
| 198 | return datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%S.%fZ') |
| 199 | |
| 200 | |
| 201 | class MyActivity(object): |
| 202 | def __init__(self, options): |
| 203 | self.options = options |
| 204 | self.modified_after = options.begin |
| 205 | self.modified_before = options.end |
| 206 | self.user = options.user |
| 207 | self.changes = [] |
| 208 | self.reviews = [] |
| 209 | self.issues = [] |
| 210 | self.check_cookies() |
| 211 | self.google_code_auth_token = None |
| 212 | |
| 213 | # Check the codereview cookie jar to determine which Rietveld instances to |
| 214 | # authenticate to. |
| 215 | def check_cookies(self): |
| 216 | cookie_file = os.path.expanduser('~/.codereview_upload_cookies') |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 217 | if not os.path.exists(cookie_file): |
deymo@chromium.org | 8aee486 | 2013-11-13 19:33:27 +0000 | [diff] [blame] | 218 | print 'No Rietveld cookie file found.' |
| 219 | cookie_jar = [] |
| 220 | else: |
| 221 | cookie_jar = cookielib.MozillaCookieJar(cookie_file) |
| 222 | try: |
| 223 | cookie_jar.load() |
| 224 | print 'Found cookie file: %s' % cookie_file |
| 225 | except (cookielib.LoadError, IOError): |
| 226 | print 'Error loading Rietveld cookie file: %s' % cookie_file |
| 227 | cookie_jar = [] |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 228 | |
| 229 | filtered_instances = [] |
| 230 | |
| 231 | def has_cookie(instance): |
| 232 | for cookie in cookie_jar: |
| 233 | if cookie.name == 'SACSID' and cookie.domain == instance['url']: |
| 234 | return True |
| 235 | if self.options.auth: |
| 236 | return get_yes_or_no('No cookie found for %s. Authorize for this ' |
| 237 | 'instance? (may require application-specific ' |
| 238 | 'password)' % instance['url']) |
| 239 | filtered_instances.append(instance) |
| 240 | return False |
| 241 | |
| 242 | for instance in rietveld_instances: |
| 243 | instance['auth'] = has_cookie(instance) |
| 244 | |
| 245 | if filtered_instances: |
| 246 | print ('No cookie found for the following Rietveld instance%s:' % |
| 247 | ('s' if len(filtered_instances) > 1 else '')) |
| 248 | for instance in filtered_instances: |
| 249 | print '\t' + instance['url'] |
| 250 | print 'Use --auth if you would like to authenticate to them.\n' |
| 251 | |
| 252 | def rietveld_search(self, instance, owner=None, reviewer=None): |
| 253 | if instance['requires_auth'] and not instance['auth']: |
| 254 | return [] |
| 255 | |
| 256 | |
| 257 | email = None if instance['auth'] else '' |
| 258 | remote = rietveld.Rietveld('https://' + instance['url'], email, None) |
| 259 | |
| 260 | # See def search() in rietveld.py to see all the filters you can use. |
| 261 | query_modified_after = None |
| 262 | |
| 263 | if instance['supports_owner_modified_query']: |
| 264 | query_modified_after = self.modified_after.strftime('%Y-%m-%d') |
| 265 | |
| 266 | # Rietveld does not allow search by both created_before and modified_after. |
| 267 | # (And some instances don't allow search by both owner and modified_after) |
| 268 | owner_email = None |
| 269 | reviewer_email = None |
| 270 | if owner: |
| 271 | owner_email = owner + '@' + instance['email_domain'] |
| 272 | if reviewer: |
| 273 | reviewer_email = reviewer + '@' + instance['email_domain'] |
| 274 | issues = remote.search( |
| 275 | owner=owner_email, |
| 276 | reviewer=reviewer_email, |
| 277 | modified_after=query_modified_after, |
| 278 | with_messages=True) |
| 279 | |
| 280 | issues = filter( |
| 281 | lambda i: (datetime_from_rietveld(i['created']) < self.modified_before), |
| 282 | issues) |
| 283 | issues = filter( |
| 284 | lambda i: (datetime_from_rietveld(i['modified']) > self.modified_after), |
| 285 | issues) |
| 286 | |
| 287 | should_filter_by_user = True |
| 288 | issues = map(partial(self.process_rietveld_issue, instance), issues) |
| 289 | issues = filter( |
| 290 | partial(self.filter_issue, should_filter_by_user=should_filter_by_user), |
| 291 | issues) |
| 292 | issues = sorted(issues, key=lambda i: i['modified'], reverse=True) |
| 293 | |
| 294 | return issues |
| 295 | |
| 296 | def process_rietveld_issue(self, instance, issue): |
| 297 | ret = {} |
| 298 | ret['owner'] = issue['owner_email'] |
| 299 | ret['author'] = ret['owner'] |
| 300 | |
cjhopman@chromium.org | 53c1e56 | 2013-03-11 20:02:38 +0000 | [diff] [blame] | 301 | ret['reviewers'] = set(issue['reviewers']) |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 302 | |
| 303 | shorturl = instance['url'] |
| 304 | if 'shorturl' in instance: |
| 305 | shorturl = instance['shorturl'] |
| 306 | |
| 307 | ret['review_url'] = 'http://%s/%d' % (shorturl, issue['issue']) |
cjhopman@chromium.org | 53c1e56 | 2013-03-11 20:02:38 +0000 | [diff] [blame] | 308 | |
| 309 | # Rietveld sometimes has '\r\n' instead of '\n'. |
| 310 | ret['header'] = issue['description'].replace('\r', '').split('\n')[0] |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 311 | |
| 312 | ret['modified'] = datetime_from_rietveld(issue['modified']) |
| 313 | ret['created'] = datetime_from_rietveld(issue['created']) |
| 314 | ret['replies'] = self.process_rietveld_replies(issue['messages']) |
| 315 | |
| 316 | return ret |
| 317 | |
| 318 | @staticmethod |
| 319 | def process_rietveld_replies(replies): |
| 320 | ret = [] |
| 321 | for reply in replies: |
| 322 | r = {} |
| 323 | r['author'] = reply['sender'] |
| 324 | r['created'] = datetime_from_rietveld(reply['date']) |
| 325 | r['content'] = '' |
| 326 | ret.append(r) |
| 327 | return ret |
| 328 | |
deymo@chromium.org | 6c03920 | 2013-09-12 12:28:12 +0000 | [diff] [blame] | 329 | @staticmethod |
| 330 | def gerrit_changes_over_ssh(instance, filters): |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 331 | # See https://review.openstack.org/Documentation/cmd-query.html |
| 332 | # Gerrit doesn't allow filtering by created time, only modified time. |
deymo@chromium.org | 6c03920 | 2013-09-12 12:28:12 +0000 | [diff] [blame] | 333 | gquery_cmd = ['ssh', '-p', str(instance['port']), instance['host'], |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 334 | 'gerrit', 'query', |
| 335 | '--format', 'JSON', |
| 336 | '--comments', |
deymo@chromium.org | 6c03920 | 2013-09-12 12:28:12 +0000 | [diff] [blame] | 337 | '--'] + filters |
| 338 | (stdout, _) = subprocess.Popen(gquery_cmd, stdout=subprocess.PIPE, |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 339 | stderr=subprocess.PIPE).communicate() |
deymo@chromium.org | 6c03920 | 2013-09-12 12:28:12 +0000 | [diff] [blame] | 340 | # Drop the last line of the output with the stats. |
| 341 | issues = stdout.splitlines()[:-1] |
| 342 | return map(json.loads, issues) |
| 343 | |
| 344 | @staticmethod |
| 345 | def gerrit_changes_over_rest(instance, filters): |
deymo@chromium.org | f8be276 | 2013-11-06 01:01:59 +0000 | [diff] [blame] | 346 | # Convert the "key:value" filter to a dictionary. |
| 347 | req = dict(f.split(':', 1) for f in filters) |
deymo@chromium.org | 6c03920 | 2013-09-12 12:28:12 +0000 | [diff] [blame] | 348 | try: |
deymo@chromium.org | f8be276 | 2013-11-06 01:01:59 +0000 | [diff] [blame] | 349 | # Instantiate the generator to force all the requests now and catch the |
| 350 | # errors here. |
| 351 | return list(gerrit_util.GenerateAllChanges(instance['url'], req, |
| 352 | o_params=['MESSAGES', 'LABELS', 'DETAILED_ACCOUNTS'])) |
| 353 | except gerrit_util.GerritError, e: |
| 354 | print 'ERROR: Looking up %r: %s' % (instance['url'], e) |
deymo@chromium.org | 6c03920 | 2013-09-12 12:28:12 +0000 | [diff] [blame] | 355 | return [] |
| 356 | |
deymo@chromium.org | 6c03920 | 2013-09-12 12:28:12 +0000 | [diff] [blame] | 357 | def gerrit_search(self, instance, owner=None, reviewer=None): |
| 358 | max_age = datetime.today() - self.modified_after |
| 359 | max_age = max_age.days * 24 * 3600 + max_age.seconds |
| 360 | user_filter = 'owner:%s' % owner if owner else 'reviewer:%s' % reviewer |
| 361 | filters = ['-age:%ss' % max_age, user_filter] |
| 362 | |
| 363 | # Determine the gerrit interface to use: SSH or REST API: |
| 364 | if 'host' in instance: |
| 365 | issues = self.gerrit_changes_over_ssh(instance, filters) |
| 366 | issues = [self.process_gerrit_ssh_issue(instance, issue) |
| 367 | for issue in issues] |
| 368 | elif 'url' in instance: |
| 369 | issues = self.gerrit_changes_over_rest(instance, filters) |
| 370 | issues = [self.process_gerrit_rest_issue(instance, issue) |
| 371 | for issue in issues] |
| 372 | else: |
| 373 | raise Exception('Invalid gerrit_instances configuration.') |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 374 | |
| 375 | # TODO(cjhopman): should we filter abandoned changes? |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 376 | issues = filter(self.filter_issue, issues) |
| 377 | issues = sorted(issues, key=lambda i: i['modified'], reverse=True) |
| 378 | |
| 379 | return issues |
| 380 | |
deymo@chromium.org | 6c03920 | 2013-09-12 12:28:12 +0000 | [diff] [blame] | 381 | def process_gerrit_ssh_issue(self, instance, issue): |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 382 | ret = {} |
| 383 | ret['review_url'] = issue['url'] |
deymo@chromium.org | e52bd5a | 2013-08-29 18:05:21 +0000 | [diff] [blame] | 384 | if 'shorturl' in instance: |
| 385 | ret['review_url'] = 'http://%s/%s' % (instance['shorturl'], |
| 386 | issue['number']) |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 387 | ret['header'] = issue['subject'] |
| 388 | ret['owner'] = issue['owner']['email'] |
| 389 | ret['author'] = ret['owner'] |
| 390 | ret['created'] = datetime.fromtimestamp(issue['createdOn']) |
| 391 | ret['modified'] = datetime.fromtimestamp(issue['lastUpdated']) |
| 392 | if 'comments' in issue: |
deymo@chromium.org | 6c03920 | 2013-09-12 12:28:12 +0000 | [diff] [blame] | 393 | ret['replies'] = self.process_gerrit_ssh_issue_replies(issue['comments']) |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 394 | else: |
| 395 | ret['replies'] = [] |
deymo@chromium.org | 6c03920 | 2013-09-12 12:28:12 +0000 | [diff] [blame] | 396 | ret['reviewers'] = set(r['author'] for r in ret['replies']) |
| 397 | ret['reviewers'].discard(ret['author']) |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 398 | return ret |
| 399 | |
| 400 | @staticmethod |
deymo@chromium.org | 6c03920 | 2013-09-12 12:28:12 +0000 | [diff] [blame] | 401 | def process_gerrit_ssh_issue_replies(replies): |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 402 | ret = [] |
| 403 | replies = filter(lambda r: 'email' in r['reviewer'], replies) |
| 404 | for reply in replies: |
deymo@chromium.org | 6c03920 | 2013-09-12 12:28:12 +0000 | [diff] [blame] | 405 | ret.append({ |
| 406 | 'author': reply['reviewer']['email'], |
| 407 | 'created': datetime.fromtimestamp(reply['timestamp']), |
| 408 | 'content': '', |
| 409 | }) |
| 410 | return ret |
| 411 | |
| 412 | def process_gerrit_rest_issue(self, instance, issue): |
| 413 | ret = {} |
| 414 | ret['review_url'] = 'https://%s/%s' % (instance['url'], issue['_number']) |
| 415 | if 'shorturl' in instance: |
| 416 | # TODO(deymo): Move this short link to https once crosreview.com supports |
| 417 | # it. |
| 418 | ret['review_url'] = 'http://%s/%s' % (instance['shorturl'], |
| 419 | issue['_number']) |
| 420 | ret['header'] = issue['subject'] |
| 421 | ret['owner'] = issue['owner']['email'] |
| 422 | ret['author'] = ret['owner'] |
| 423 | ret['created'] = datetime_from_gerrit(issue['created']) |
| 424 | ret['modified'] = datetime_from_gerrit(issue['updated']) |
| 425 | if 'messages' in issue: |
| 426 | ret['replies'] = self.process_gerrit_rest_issue_replies(issue['messages']) |
| 427 | else: |
| 428 | ret['replies'] = [] |
| 429 | ret['reviewers'] = set(r['author'] for r in ret['replies']) |
| 430 | ret['reviewers'].discard(ret['author']) |
| 431 | return ret |
| 432 | |
| 433 | @staticmethod |
| 434 | def process_gerrit_rest_issue_replies(replies): |
| 435 | ret = [] |
deymo@chromium.org | f8be276 | 2013-11-06 01:01:59 +0000 | [diff] [blame] | 436 | replies = filter(lambda r: 'author' in r and 'email' in r['author'], |
| 437 | replies) |
deymo@chromium.org | 6c03920 | 2013-09-12 12:28:12 +0000 | [diff] [blame] | 438 | for reply in replies: |
| 439 | ret.append({ |
| 440 | 'author': reply['author']['email'], |
| 441 | 'created': datetime_from_gerrit(reply['date']), |
| 442 | 'content': reply['message'], |
| 443 | }) |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 444 | return ret |
| 445 | |
| 446 | def google_code_issue_search(self, instance): |
| 447 | time_format = '%Y-%m-%dT%T' |
| 448 | # See http://code.google.com/p/support/wiki/IssueTrackerAPI |
| 449 | # q=<owner>@chromium.org does a full text search for <owner>@chromium.org. |
| 450 | # This will accept the issue if owner is the owner or in the cc list. Might |
| 451 | # have some false positives, though. |
| 452 | |
| 453 | # Don't filter normally on modified_before because it can filter out things |
| 454 | # that were modified in the time period and then modified again after it. |
| 455 | gcode_url = ('https://code.google.com/feeds/issues/p/%s/issues/full' % |
| 456 | instance['name']) |
| 457 | |
| 458 | gcode_data = urllib.urlencode({ |
| 459 | 'alt': 'json', |
| 460 | 'max-results': '100000', |
| 461 | 'q': '%s' % self.user, |
| 462 | 'published-max': self.modified_before.strftime(time_format), |
| 463 | 'updated-min': self.modified_after.strftime(time_format), |
| 464 | }) |
| 465 | |
| 466 | opener = urllib2.build_opener() |
cjhopman@chromium.org | 3365f2d | 2012-11-01 18:53:13 +0000 | [diff] [blame] | 467 | if self.google_code_auth_token: |
| 468 | opener.addheaders = [('Authorization', 'GoogleLogin auth=%s' % |
| 469 | self.google_code_auth_token)] |
| 470 | gcode_json = None |
| 471 | try: |
| 472 | gcode_get = opener.open(gcode_url + '?' + gcode_data) |
| 473 | gcode_json = json.load(gcode_get) |
| 474 | gcode_get.close() |
| 475 | except urllib2.HTTPError, _: |
| 476 | print 'Unable to access ' + instance['name'] + ' issue tracker.' |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 477 | |
cjhopman@chromium.org | 3365f2d | 2012-11-01 18:53:13 +0000 | [diff] [blame] | 478 | if not gcode_json or 'entry' not in gcode_json['feed']: |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 479 | return [] |
| 480 | |
| 481 | issues = gcode_json['feed']['entry'] |
| 482 | issues = map(partial(self.process_google_code_issue, instance), issues) |
| 483 | issues = filter(self.filter_issue, issues) |
| 484 | issues = sorted(issues, key=lambda i: i['modified'], reverse=True) |
| 485 | return issues |
| 486 | |
| 487 | def process_google_code_issue(self, project, issue): |
| 488 | ret = {} |
| 489 | ret['created'] = datetime_from_google_code(issue['published']['$t']) |
| 490 | ret['modified'] = datetime_from_google_code(issue['updated']['$t']) |
| 491 | |
| 492 | ret['owner'] = '' |
nyquist@chromium.org | 18bc90d | 2012-12-20 19:26:47 +0000 | [diff] [blame] | 493 | if 'issues$owner' in issue: |
| 494 | ret['owner'] = issue['issues$owner']['issues$username']['$t'] |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 495 | ret['author'] = issue['author'][0]['name']['$t'] |
| 496 | |
| 497 | if 'shorturl' in project: |
| 498 | issue_id = issue['id']['$t'] |
| 499 | issue_id = issue_id[issue_id.rfind('/') + 1:] |
| 500 | ret['url'] = 'http://%s/%d' % (project['shorturl'], int(issue_id)) |
| 501 | else: |
| 502 | issue_url = issue['link'][1] |
| 503 | if issue_url['rel'] != 'alternate': |
| 504 | raise RuntimeError |
| 505 | ret['url'] = issue_url['href'] |
| 506 | ret['header'] = issue['title']['$t'] |
| 507 | |
| 508 | ret['replies'] = self.get_google_code_issue_replies(issue) |
| 509 | return ret |
| 510 | |
| 511 | def get_google_code_issue_replies(self, issue): |
| 512 | """Get all the comments on the issue.""" |
| 513 | replies_url = issue['link'][0] |
| 514 | if replies_url['rel'] != 'replies': |
| 515 | raise RuntimeError |
| 516 | |
| 517 | replies_data = urllib.urlencode({ |
| 518 | 'alt': 'json', |
| 519 | 'fields': 'entry(published,author,content)', |
| 520 | }) |
| 521 | |
| 522 | opener = urllib2.build_opener() |
| 523 | opener.addheaders = [('Authorization', 'GoogleLogin auth=%s' % |
| 524 | self.google_code_auth_token)] |
| 525 | try: |
| 526 | replies_get = opener.open(replies_url['href'] + '?' + replies_data) |
| 527 | except urllib2.HTTPError, _: |
| 528 | return [] |
| 529 | |
| 530 | replies_json = json.load(replies_get) |
| 531 | replies_get.close() |
| 532 | return self.process_google_code_issue_replies(replies_json) |
| 533 | |
| 534 | @staticmethod |
| 535 | def process_google_code_issue_replies(replies): |
| 536 | if 'entry' not in replies['feed']: |
| 537 | return [] |
| 538 | |
| 539 | ret = [] |
| 540 | for entry in replies['feed']['entry']: |
| 541 | e = {} |
| 542 | e['created'] = datetime_from_google_code(entry['published']['$t']) |
| 543 | e['content'] = entry['content']['$t'] |
| 544 | e['author'] = entry['author'][0]['name']['$t'] |
| 545 | ret.append(e) |
| 546 | return ret |
| 547 | |
jsbell@chromium.org | c92f582 | 2014-01-06 23:49:11 +0000 | [diff] [blame] | 548 | def print_heading(self, heading): |
| 549 | print |
| 550 | print self.options.output_format_heading.format(heading=heading) |
| 551 | |
nyquist@chromium.org | 18bc90d | 2012-12-20 19:26:47 +0000 | [diff] [blame] | 552 | def print_change(self, change): |
cjhopman@chromium.org | 53c1e56 | 2013-03-11 20:02:38 +0000 | [diff] [blame] | 553 | optional_values = { |
| 554 | 'reviewers': ', '.join(change['reviewers']) |
| 555 | } |
nyquist@chromium.org | 18bc90d | 2012-12-20 19:26:47 +0000 | [diff] [blame] | 556 | self.print_generic(self.options.output_format, |
| 557 | self.options.output_format_changes, |
| 558 | change['header'], |
| 559 | change['review_url'], |
cjhopman@chromium.org | 53c1e56 | 2013-03-11 20:02:38 +0000 | [diff] [blame] | 560 | change['author'], |
| 561 | optional_values) |
nyquist@chromium.org | 18bc90d | 2012-12-20 19:26:47 +0000 | [diff] [blame] | 562 | |
| 563 | def print_issue(self, issue): |
| 564 | optional_values = { |
| 565 | 'owner': issue['owner'], |
| 566 | } |
| 567 | self.print_generic(self.options.output_format, |
| 568 | self.options.output_format_issues, |
| 569 | issue['header'], |
| 570 | issue['url'], |
| 571 | issue['author'], |
| 572 | optional_values) |
| 573 | |
| 574 | def print_review(self, review): |
| 575 | self.print_generic(self.options.output_format, |
| 576 | self.options.output_format_reviews, |
| 577 | review['header'], |
| 578 | review['review_url'], |
| 579 | review['author']) |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 580 | |
| 581 | @staticmethod |
nyquist@chromium.org | 18bc90d | 2012-12-20 19:26:47 +0000 | [diff] [blame] | 582 | def print_generic(default_fmt, specific_fmt, |
| 583 | title, url, author, |
| 584 | optional_values=None): |
| 585 | output_format = specific_fmt if specific_fmt is not None else default_fmt |
| 586 | output_format = unicode(output_format) |
| 587 | required_values = { |
| 588 | 'title': title, |
| 589 | 'url': url, |
| 590 | 'author': author, |
| 591 | } |
| 592 | # Merge required and optional values. |
| 593 | if optional_values is not None: |
| 594 | values = dict(required_values.items() + optional_values.items()) |
| 595 | else: |
| 596 | values = required_values |
| 597 | print output_format.format(**values) |
| 598 | |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 599 | |
| 600 | def filter_issue(self, issue, should_filter_by_user=True): |
| 601 | def maybe_filter_username(email): |
| 602 | return not should_filter_by_user or username(email) == self.user |
| 603 | if (maybe_filter_username(issue['author']) and |
| 604 | self.filter_modified(issue['created'])): |
| 605 | return True |
| 606 | if (maybe_filter_username(issue['owner']) and |
| 607 | (self.filter_modified(issue['created']) or |
| 608 | self.filter_modified(issue['modified']))): |
| 609 | return True |
| 610 | for reply in issue['replies']: |
| 611 | if self.filter_modified(reply['created']): |
| 612 | if not should_filter_by_user: |
| 613 | break |
| 614 | if (username(reply['author']) == self.user |
| 615 | or (self.user + '@') in reply['content']): |
| 616 | break |
| 617 | else: |
| 618 | return False |
| 619 | return True |
| 620 | |
| 621 | def filter_modified(self, modified): |
| 622 | return self.modified_after < modified and modified < self.modified_before |
| 623 | |
| 624 | def auth_for_changes(self): |
| 625 | #TODO(cjhopman): Move authentication check for getting changes here. |
| 626 | pass |
| 627 | |
| 628 | def auth_for_reviews(self): |
| 629 | # Reviews use all the same instances as changes so no authentication is |
| 630 | # required. |
| 631 | pass |
| 632 | |
| 633 | def auth_for_issues(self): |
| 634 | self.google_code_auth_token = ( |
| 635 | get_auth_token(self.options.local_user + '@chromium.org')) |
| 636 | |
| 637 | def get_changes(self): |
| 638 | for instance in rietveld_instances: |
| 639 | self.changes += self.rietveld_search(instance, owner=self.user) |
| 640 | |
| 641 | for instance in gerrit_instances: |
| 642 | self.changes += self.gerrit_search(instance, owner=self.user) |
| 643 | |
| 644 | def print_changes(self): |
| 645 | if self.changes: |
jsbell@chromium.org | c92f582 | 2014-01-06 23:49:11 +0000 | [diff] [blame] | 646 | self.print_heading('Changes') |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 647 | for change in self.changes: |
| 648 | self.print_change(change) |
| 649 | |
| 650 | def get_reviews(self): |
| 651 | for instance in rietveld_instances: |
| 652 | self.reviews += self.rietveld_search(instance, reviewer=self.user) |
| 653 | |
| 654 | for instance in gerrit_instances: |
| 655 | reviews = self.gerrit_search(instance, reviewer=self.user) |
| 656 | reviews = filter(lambda r: not username(r['owner']) == self.user, reviews) |
| 657 | self.reviews += reviews |
| 658 | |
| 659 | def print_reviews(self): |
| 660 | if self.reviews: |
jsbell@chromium.org | c92f582 | 2014-01-06 23:49:11 +0000 | [diff] [blame] | 661 | self.print_heading('Reviews') |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 662 | for review in self.reviews: |
nyquist@chromium.org | 18bc90d | 2012-12-20 19:26:47 +0000 | [diff] [blame] | 663 | self.print_review(review) |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 664 | |
| 665 | def get_issues(self): |
| 666 | for project in google_code_projects: |
| 667 | self.issues += self.google_code_issue_search(project) |
| 668 | |
nyquist@chromium.org | 18bc90d | 2012-12-20 19:26:47 +0000 | [diff] [blame] | 669 | def print_issues(self): |
| 670 | if self.issues: |
jsbell@chromium.org | c92f582 | 2014-01-06 23:49:11 +0000 | [diff] [blame] | 671 | self.print_heading('Issues') |
nyquist@chromium.org | 18bc90d | 2012-12-20 19:26:47 +0000 | [diff] [blame] | 672 | for issue in self.issues: |
| 673 | self.print_issue(issue) |
| 674 | |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 675 | def print_activity(self): |
| 676 | self.print_changes() |
| 677 | self.print_reviews() |
| 678 | self.print_issues() |
| 679 | |
| 680 | |
| 681 | def main(): |
| 682 | # Silence upload.py. |
| 683 | rietveld.upload.verbosity = 0 |
| 684 | |
| 685 | parser = optparse.OptionParser(description=sys.modules[__name__].__doc__) |
| 686 | parser.add_option( |
| 687 | '-u', '--user', metavar='<email>', |
| 688 | default=os.environ.get('USER'), |
| 689 | help='Filter on user, default=%default') |
| 690 | parser.add_option( |
| 691 | '-b', '--begin', metavar='<date>', |
| 692 | help='Filter issues created after the date') |
| 693 | parser.add_option( |
| 694 | '-e', '--end', metavar='<date>', |
| 695 | help='Filter issues created before the date') |
| 696 | quarter_begin, quarter_end = get_quarter_of(datetime.today() - |
| 697 | relativedelta(months=2)) |
| 698 | parser.add_option( |
| 699 | '-Q', '--last_quarter', action='store_true', |
| 700 | help='Use last quarter\'s dates, e.g. %s to %s' % ( |
| 701 | quarter_begin.strftime('%Y-%m-%d'), quarter_end.strftime('%Y-%m-%d'))) |
| 702 | parser.add_option( |
| 703 | '-Y', '--this_year', action='store_true', |
| 704 | help='Use this year\'s dates') |
| 705 | parser.add_option( |
| 706 | '-w', '--week_of', metavar='<date>', |
| 707 | help='Show issues for week of the date') |
| 708 | parser.add_option( |
| 709 | '-a', '--auth', |
| 710 | action='store_true', |
| 711 | help='Ask to authenticate for instances with no auth cookie') |
| 712 | |
nyquist@chromium.org | 18bc90d | 2012-12-20 19:26:47 +0000 | [diff] [blame] | 713 | activity_types_group = optparse.OptionGroup(parser, 'Activity Types', |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 714 | 'By default, all activity will be looked up and ' |
| 715 | 'printed. If any of these are specified, only ' |
| 716 | 'those specified will be searched.') |
nyquist@chromium.org | 18bc90d | 2012-12-20 19:26:47 +0000 | [diff] [blame] | 717 | activity_types_group.add_option( |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 718 | '-c', '--changes', |
| 719 | action='store_true', |
| 720 | help='Show changes.') |
nyquist@chromium.org | 18bc90d | 2012-12-20 19:26:47 +0000 | [diff] [blame] | 721 | activity_types_group.add_option( |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 722 | '-i', '--issues', |
| 723 | action='store_true', |
| 724 | help='Show issues.') |
nyquist@chromium.org | 18bc90d | 2012-12-20 19:26:47 +0000 | [diff] [blame] | 725 | activity_types_group.add_option( |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 726 | '-r', '--reviews', |
| 727 | action='store_true', |
| 728 | help='Show reviews.') |
nyquist@chromium.org | 18bc90d | 2012-12-20 19:26:47 +0000 | [diff] [blame] | 729 | parser.add_option_group(activity_types_group) |
| 730 | |
| 731 | output_format_group = optparse.OptionGroup(parser, 'Output Format', |
| 732 | 'By default, all activity will be printed in the ' |
| 733 | 'following format: {url} {title}. This can be ' |
| 734 | 'changed for either all activity types or ' |
| 735 | 'individually for each activity type. The format ' |
| 736 | 'is defined as documented for ' |
| 737 | 'string.format(...). The variables available for ' |
| 738 | 'all activity types are url, title and author. ' |
| 739 | 'Format options for specific activity types will ' |
| 740 | 'override the generic format.') |
| 741 | output_format_group.add_option( |
| 742 | '-f', '--output-format', metavar='<format>', |
| 743 | default=u'{url} {title}', |
| 744 | help='Specifies the format to use when printing all your activity.') |
| 745 | output_format_group.add_option( |
| 746 | '--output-format-changes', metavar='<format>', |
| 747 | default=None, |
cjhopman@chromium.org | 53c1e56 | 2013-03-11 20:02:38 +0000 | [diff] [blame] | 748 | help='Specifies the format to use when printing changes. Supports the ' |
| 749 | 'additional variable {reviewers}') |
nyquist@chromium.org | 18bc90d | 2012-12-20 19:26:47 +0000 | [diff] [blame] | 750 | output_format_group.add_option( |
| 751 | '--output-format-issues', metavar='<format>', |
| 752 | default=None, |
cjhopman@chromium.org | 53c1e56 | 2013-03-11 20:02:38 +0000 | [diff] [blame] | 753 | help='Specifies the format to use when printing issues. Supports the ' |
| 754 | 'additional variable {owner}.') |
nyquist@chromium.org | 18bc90d | 2012-12-20 19:26:47 +0000 | [diff] [blame] | 755 | output_format_group.add_option( |
| 756 | '--output-format-reviews', metavar='<format>', |
| 757 | default=None, |
| 758 | help='Specifies the format to use when printing reviews.') |
jsbell@chromium.org | c92f582 | 2014-01-06 23:49:11 +0000 | [diff] [blame] | 759 | output_format_group.add_option( |
| 760 | '--output-format-heading', metavar='<format>', |
| 761 | default=u'{heading}:', |
| 762 | help='Specifies the format to use when printing headings.') |
| 763 | output_format_group.add_option( |
| 764 | '-m', '--markdown', action='store_true', |
| 765 | help='Use markdown-friendly output (overrides --output-format ' |
| 766 | 'and --output-format-heading)') |
nyquist@chromium.org | 18bc90d | 2012-12-20 19:26:47 +0000 | [diff] [blame] | 767 | parser.add_option_group(output_format_group) |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 768 | |
| 769 | # Remove description formatting |
| 770 | parser.format_description = ( |
| 771 | lambda _: parser.description) # pylint: disable=E1101 |
| 772 | |
| 773 | options, args = parser.parse_args() |
| 774 | options.local_user = os.environ.get('USER') |
| 775 | if args: |
| 776 | parser.error('Args unsupported') |
| 777 | if not options.user: |
| 778 | parser.error('USER is not set, please use -u') |
| 779 | |
| 780 | options.user = username(options.user) |
| 781 | |
| 782 | if not options.begin: |
| 783 | if options.last_quarter: |
| 784 | begin, end = quarter_begin, quarter_end |
| 785 | elif options.this_year: |
| 786 | begin, end = get_year_of(datetime.today()) |
| 787 | elif options.week_of: |
| 788 | begin, end = (get_week_of(datetime.strptime(options.week_of, '%m/%d/%y'))) |
| 789 | else: |
| 790 | begin, end = (get_week_of(datetime.today() - timedelta(days=1))) |
| 791 | else: |
| 792 | begin = datetime.strptime(options.begin, '%m/%d/%y') |
| 793 | if options.end: |
| 794 | end = datetime.strptime(options.end, '%m/%d/%y') |
| 795 | else: |
| 796 | end = datetime.today() |
| 797 | options.begin, options.end = begin, end |
| 798 | |
jsbell@chromium.org | c92f582 | 2014-01-06 23:49:11 +0000 | [diff] [blame] | 799 | if options.markdown: |
| 800 | options.output_format = ' * [{title}]({url})' |
| 801 | options.output_format_heading = '### {heading} ###' |
| 802 | |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 803 | print 'Searching for activity by %s' % options.user |
| 804 | print 'Using range %s to %s' % (options.begin, options.end) |
| 805 | |
| 806 | my_activity = MyActivity(options) |
| 807 | |
| 808 | if not (options.changes or options.reviews or options.issues): |
| 809 | options.changes = True |
| 810 | options.issues = True |
| 811 | options.reviews = True |
| 812 | |
| 813 | # First do any required authentication so none of the user interaction has to |
| 814 | # wait for actual work. |
| 815 | if options.changes: |
| 816 | my_activity.auth_for_changes() |
| 817 | if options.reviews: |
| 818 | my_activity.auth_for_reviews() |
| 819 | if options.issues: |
| 820 | my_activity.auth_for_issues() |
| 821 | |
| 822 | print 'Looking up activity.....' |
| 823 | |
| 824 | if options.changes: |
| 825 | my_activity.get_changes() |
| 826 | if options.reviews: |
| 827 | my_activity.get_reviews() |
| 828 | if options.issues: |
| 829 | my_activity.get_issues() |
| 830 | |
cjhopman@chromium.org | 04d119d | 2012-10-17 22:41:53 +0000 | [diff] [blame] | 831 | print '\n\n\n' |
| 832 | |
| 833 | my_activity.print_changes() |
| 834 | my_activity.print_reviews() |
| 835 | my_activity.print_issues() |
| 836 | return 0 |
| 837 | |
| 838 | |
| 839 | if __name__ == '__main__': |
| 840 | sys.exit(main()) |