blob: 19b303b5b9ddae6ce8f5f0bf938420c6fe138dbf [file] [log] [blame]
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +00001#!/usr/bin/env python
2# Copyright (c) 2012 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Get stats about your activity.
7
8Example:
9 - my_activity.py for stats for the current week (last week on mondays).
10 - my_activity.py -Q for stats for last quarter.
11 - my_activity.py -Y for stats for this year.
12 - my_activity.py -b 4/5/12 for stats since 4/5/12.
13 - my_activity.py -b 4/5/12 -e 6/7/12 for stats between 4/5/12 and 6/7/12.
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +000014 - my_activity.py -jd to output stats for the week to json with deltas data.
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +000015"""
16
vadimsh@chromium.orgcf6a5d22015-04-09 22:02:00 +000017# TODO(vadimsh): This script knows too much about ClientLogin and cookies. It
18# will stop to work on ~20 Apr 2015.
19
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +000020# These services typically only provide a created time and a last modified time
21# for each item for general queries. This is not enough to determine if there
22# was activity in a given time period. So, we first query for all things created
23# before end and modified after begin. Then, we get the details of each item and
24# check those details to determine if there was activity in the given period.
25# This means that query time scales mostly with (today() - begin).
26
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +000027from datetime import datetime
28from datetime import timedelta
29from functools import partial
30import json
Tobias Sargeantffb3c432017-03-08 14:09:14 +000031import logging
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +000032import optparse
33import os
34import subprocess
Tobias Sargeantffb3c432017-03-08 14:09:14 +000035from string import Formatter
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +000036import sys
37import urllib
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +000038import re
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +000039
vadimsh@chromium.orgcf6a5d22015-04-09 22:02:00 +000040import auth
stevefung@chromium.org832d51e2015-05-27 12:52:51 +000041import fix_encoding
deymo@chromium.orgf8be2762013-11-06 01:01:59 +000042import gerrit_util
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +000043import rietveld
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +000044
seanmccullough@chromium.org3e4a5812015-06-11 17:48:47 +000045from third_party import httplib2
46
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +000047try:
Tobias Sargeantffb3c432017-03-08 14:09:14 +000048 import dateutil # pylint: disable=import-error
49 import dateutil.parser
50 from dateutil.relativedelta import relativedelta
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +000051except ImportError:
Tobias Sargeantffb3c432017-03-08 14:09:14 +000052 logging.error('python-dateutil package required')
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +000053 exit(1)
54
Tobias Sargeantffb3c432017-03-08 14:09:14 +000055
56class DefaultFormatter(Formatter):
57 def __init__(self, default = ''):
58 super(DefaultFormatter, self).__init__()
59 self.default = default
60
61 def get_value(self, key, args, kwds):
62 if isinstance(key, basestring) and key not in kwds:
63 return self.default
64 return Formatter.get_value(self, key, args, kwds)
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +000065
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +000066rietveld_instances = [
67 {
68 'url': 'codereview.chromium.org',
69 'shorturl': 'crrev.com',
70 'supports_owner_modified_query': True,
71 'requires_auth': False,
72 'email_domain': 'chromium.org',
73 },
74 {
75 'url': 'chromereviews.googleplex.com',
76 'shorturl': 'go/chromerev',
77 'supports_owner_modified_query': True,
78 'requires_auth': True,
79 'email_domain': 'google.com',
80 },
81 {
82 'url': 'codereview.appspot.com',
83 'supports_owner_modified_query': True,
84 'requires_auth': False,
85 'email_domain': 'chromium.org',
86 },
87 {
88 'url': 'breakpad.appspot.com',
89 'supports_owner_modified_query': False,
90 'requires_auth': False,
91 'email_domain': 'chromium.org',
92 },
93]
94
95gerrit_instances = [
96 {
deymo@chromium.org6c039202013-09-12 12:28:12 +000097 'url': 'chromium-review.googlesource.com',
Jeremy Romanf475a472017-06-06 16:49:11 -040098 'shorturl': 'crrev.com/c',
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +000099 },
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000100 {
101 'url': 'chrome-internal-review.googlesource.com',
Jeremy Romanf475a472017-06-06 16:49:11 -0400102 'shorturl': 'crrev.com/i',
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000103 },
deymo@chromium.org56dc57a2015-09-10 18:26:54 +0000104 {
105 'url': 'android-review.googlesource.com',
106 },
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000107]
108
109google_code_projects = [
110 {
111 'name': 'chromium',
112 'shorturl': 'crbug.com',
113 },
114 {
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000115 'name': 'google-breakpad',
116 },
117 {
118 'name': 'gyp',
enne@chromium.orgf01fad32012-11-26 18:09:38 +0000119 },
120 {
121 'name': 'skia',
122 },
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000123]
124
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000125def username(email):
126 """Keeps the username of an email address."""
127 return email and email.split('@', 1)[0]
128
129
cjhopman@chromium.org426557a2012-10-22 20:18:52 +0000130def datetime_to_midnight(date):
131 return date - timedelta(hours=date.hour, minutes=date.minute,
132 seconds=date.second, microseconds=date.microsecond)
133
134
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000135def get_quarter_of(date):
cjhopman@chromium.org426557a2012-10-22 20:18:52 +0000136 begin = (datetime_to_midnight(date) -
137 relativedelta(months=(date.month % 3) - 1, days=(date.day - 1)))
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000138 return begin, begin + relativedelta(months=3)
139
140
141def get_year_of(date):
cjhopman@chromium.org426557a2012-10-22 20:18:52 +0000142 begin = (datetime_to_midnight(date) -
143 relativedelta(months=(date.month - 1), days=(date.day - 1)))
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000144 return begin, begin + relativedelta(years=1)
145
146
147def get_week_of(date):
cjhopman@chromium.org426557a2012-10-22 20:18:52 +0000148 begin = (datetime_to_midnight(date) - timedelta(days=date.weekday()))
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000149 return begin, begin + timedelta(days=7)
150
151
152def get_yes_or_no(msg):
153 while True:
154 response = raw_input(msg + ' yes/no [no] ')
155 if response == 'y' or response == 'yes':
156 return True
157 elif not response or response == 'n' or response == 'no':
158 return False
159
160
deymo@chromium.org6c039202013-09-12 12:28:12 +0000161def datetime_from_gerrit(date_string):
162 return datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S.%f000')
163
164
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000165def datetime_from_rietveld(date_string):
deymo@chromium.org29eb6e62014-03-20 01:55:55 +0000166 try:
167 return datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S.%f')
168 except ValueError:
169 # Sometimes rietveld returns a value without the milliseconds part, so we
170 # attempt to parse those cases as well.
171 return datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S')
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000172
173
174def datetime_from_google_code(date_string):
175 return datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%S.%fZ')
176
177
178class MyActivity(object):
179 def __init__(self, options):
180 self.options = options
181 self.modified_after = options.begin
182 self.modified_before = options.end
183 self.user = options.user
184 self.changes = []
185 self.reviews = []
186 self.issues = []
187 self.check_cookies()
188 self.google_code_auth_token = None
189
190 # Check the codereview cookie jar to determine which Rietveld instances to
191 # authenticate to.
192 def check_cookies(self):
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000193 filtered_instances = []
194
195 def has_cookie(instance):
seanmccullough@chromium.org3e4a5812015-06-11 17:48:47 +0000196 auth_config = auth.extract_auth_config_from_options(self.options)
197 a = auth.get_authenticator_for_host(instance['url'], auth_config)
198 return a.has_cached_credentials()
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000199
200 for instance in rietveld_instances:
201 instance['auth'] = has_cookie(instance)
202
203 if filtered_instances:
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000204 logging.warning('No cookie found for the following Rietveld instance%s:',
205 's' if len(filtered_instances) > 1 else '')
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000206 for instance in filtered_instances:
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000207 logging.warning('\t' + instance['url'])
208 logging.warning('Use --auth if you would like to authenticate to them.')
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000209
210 def rietveld_search(self, instance, owner=None, reviewer=None):
211 if instance['requires_auth'] and not instance['auth']:
212 return []
213
214
215 email = None if instance['auth'] else ''
vadimsh@chromium.orgcf6a5d22015-04-09 22:02:00 +0000216 auth_config = auth.extract_auth_config_from_options(self.options)
217 remote = rietveld.Rietveld('https://' + instance['url'], auth_config, email)
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000218
219 # See def search() in rietveld.py to see all the filters you can use.
220 query_modified_after = None
221
222 if instance['supports_owner_modified_query']:
223 query_modified_after = self.modified_after.strftime('%Y-%m-%d')
224
225 # Rietveld does not allow search by both created_before and modified_after.
226 # (And some instances don't allow search by both owner and modified_after)
227 owner_email = None
228 reviewer_email = None
229 if owner:
230 owner_email = owner + '@' + instance['email_domain']
231 if reviewer:
232 reviewer_email = reviewer + '@' + instance['email_domain']
233 issues = remote.search(
234 owner=owner_email,
235 reviewer=reviewer_email,
236 modified_after=query_modified_after,
237 with_messages=True)
238
239 issues = filter(
240 lambda i: (datetime_from_rietveld(i['created']) < self.modified_before),
241 issues)
242 issues = filter(
243 lambda i: (datetime_from_rietveld(i['modified']) > self.modified_after),
244 issues)
245
246 should_filter_by_user = True
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000247 issues = map(partial(self.process_rietveld_issue, remote, instance), issues)
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000248 issues = filter(
249 partial(self.filter_issue, should_filter_by_user=should_filter_by_user),
250 issues)
251 issues = sorted(issues, key=lambda i: i['modified'], reverse=True)
252
253 return issues
254
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +0000255 def extract_bug_number_from_description(self, issue):
256 description = None
257
258 if 'description' in issue:
259 # Getting the description for Rietveld
260 description = issue['description']
261 elif 'revisions' in issue:
262 # Getting the description for REST Gerrit
263 revision = issue['revisions'][issue['current_revision']]
264 description = revision['commit']['message']
265
266 bugs = []
267 if description:
Nicolas Dossou-gbete903ea732017-07-10 16:46:59 +0100268 # Handle both "Bug: 99999" and "BUG=99999" bug notations
269 # Multiple bugs can be noted on a single line or in multiple ones.
270 matches = re.findall(r'BUG[=:]\s?(((\d+)(,\s?)?)+)', description,
271 flags=re.IGNORECASE)
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +0000272 if matches:
273 for match in matches:
274 bugs.extend(match[0].replace(' ', '').split(','))
275
276 return bugs
277
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000278 def process_rietveld_issue(self, remote, instance, issue):
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000279 ret = {}
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000280 if self.options.deltas:
281 patchset_props = remote.get_patchset_properties(
282 issue['issue'],
283 issue['patchsets'][-1])
284 ret['delta'] = '+%d,-%d' % (
285 sum(f['num_added'] for f in patchset_props['files'].itervalues()),
286 sum(f['num_removed'] for f in patchset_props['files'].itervalues()))
287
288 if issue['landed_days_ago'] != 'unknown':
289 ret['status'] = 'committed'
290 elif issue['closed']:
291 ret['status'] = 'closed'
292 elif len(issue['reviewers']) and issue['all_required_reviewers_approved']:
293 ret['status'] = 'ready'
294 else:
295 ret['status'] = 'open'
296
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000297 ret['owner'] = issue['owner_email']
298 ret['author'] = ret['owner']
299
cjhopman@chromium.org53c1e562013-03-11 20:02:38 +0000300 ret['reviewers'] = set(issue['reviewers'])
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000301
302 shorturl = instance['url']
303 if 'shorturl' in instance:
304 shorturl = instance['shorturl']
305
306 ret['review_url'] = 'http://%s/%d' % (shorturl, issue['issue'])
cjhopman@chromium.org53c1e562013-03-11 20:02:38 +0000307
308 # Rietveld sometimes has '\r\n' instead of '\n'.
309 ret['header'] = issue['description'].replace('\r', '').split('\n')[0]
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000310
311 ret['modified'] = datetime_from_rietveld(issue['modified'])
312 ret['created'] = datetime_from_rietveld(issue['created'])
313 ret['replies'] = self.process_rietveld_replies(issue['messages'])
314
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +0000315 ret['bug'] = self.extract_bug_number_from_description(issue)
316 ret['landed_days_ago'] = issue['landed_days_ago']
317
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000318 return ret
319
320 @staticmethod
321 def process_rietveld_replies(replies):
322 ret = []
323 for reply in replies:
324 r = {}
325 r['author'] = reply['sender']
326 r['created'] = datetime_from_rietveld(reply['date'])
327 r['content'] = ''
328 ret.append(r)
329 return ret
330
deymo@chromium.org6c039202013-09-12 12:28:12 +0000331 @staticmethod
332 def gerrit_changes_over_ssh(instance, filters):
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000333 # See https://review.openstack.org/Documentation/cmd-query.html
334 # Gerrit doesn't allow filtering by created time, only modified time.
deymo@chromium.org6c039202013-09-12 12:28:12 +0000335 gquery_cmd = ['ssh', '-p', str(instance['port']), instance['host'],
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000336 'gerrit', 'query',
337 '--format', 'JSON',
338 '--comments',
deymo@chromium.org6c039202013-09-12 12:28:12 +0000339 '--'] + filters
340 (stdout, _) = subprocess.Popen(gquery_cmd, stdout=subprocess.PIPE,
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000341 stderr=subprocess.PIPE).communicate()
deymo@chromium.org6c039202013-09-12 12:28:12 +0000342 # Drop the last line of the output with the stats.
343 issues = stdout.splitlines()[:-1]
344 return map(json.loads, issues)
345
346 @staticmethod
347 def gerrit_changes_over_rest(instance, filters):
Michael Achenbach6fbf12f2017-07-06 10:54:11 +0200348 # Convert the "key:value" filter to a list of (key, value) pairs.
349 req = list(f.split(':', 1) for f in filters)
deymo@chromium.org6c039202013-09-12 12:28:12 +0000350 try:
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000351 # Instantiate the generator to force all the requests now and catch the
352 # errors here.
353 return list(gerrit_util.GenerateAllChanges(instance['url'], req,
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +0000354 o_params=['MESSAGES', 'LABELS', 'DETAILED_ACCOUNTS',
355 'CURRENT_REVISION', 'CURRENT_COMMIT']))
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000356 except gerrit_util.GerritError, e:
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000357 logging.error('Looking up %r: %s', instance['url'], e)
deymo@chromium.org6c039202013-09-12 12:28:12 +0000358 return []
359
deymo@chromium.org6c039202013-09-12 12:28:12 +0000360 def gerrit_search(self, instance, owner=None, reviewer=None):
361 max_age = datetime.today() - self.modified_after
362 max_age = max_age.days * 24 * 3600 + max_age.seconds
363 user_filter = 'owner:%s' % owner if owner else 'reviewer:%s' % reviewer
364 filters = ['-age:%ss' % max_age, user_filter]
365
366 # Determine the gerrit interface to use: SSH or REST API:
367 if 'host' in instance:
368 issues = self.gerrit_changes_over_ssh(instance, filters)
369 issues = [self.process_gerrit_ssh_issue(instance, issue)
370 for issue in issues]
371 elif 'url' in instance:
372 issues = self.gerrit_changes_over_rest(instance, filters)
373 issues = [self.process_gerrit_rest_issue(instance, issue)
374 for issue in issues]
375 else:
376 raise Exception('Invalid gerrit_instances configuration.')
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000377
378 # TODO(cjhopman): should we filter abandoned changes?
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000379 issues = filter(self.filter_issue, issues)
380 issues = sorted(issues, key=lambda i: i['modified'], reverse=True)
381
382 return issues
383
deymo@chromium.org6c039202013-09-12 12:28:12 +0000384 def process_gerrit_ssh_issue(self, instance, issue):
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000385 ret = {}
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000386 if self.options.deltas:
387 ret['delta'] = DefaultFormatter().format(
388 '+{insertions},-{deletions}',
389 **issue)
390 ret['status'] = issue['status']
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000391 ret['review_url'] = issue['url']
deymo@chromium.orge52bd5a2013-08-29 18:05:21 +0000392 if 'shorturl' in instance:
393 ret['review_url'] = 'http://%s/%s' % (instance['shorturl'],
394 issue['number'])
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000395 ret['header'] = issue['subject']
396 ret['owner'] = issue['owner']['email']
397 ret['author'] = ret['owner']
398 ret['created'] = datetime.fromtimestamp(issue['createdOn'])
399 ret['modified'] = datetime.fromtimestamp(issue['lastUpdated'])
400 if 'comments' in issue:
deymo@chromium.org6c039202013-09-12 12:28:12 +0000401 ret['replies'] = self.process_gerrit_ssh_issue_replies(issue['comments'])
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000402 else:
403 ret['replies'] = []
deymo@chromium.org6c039202013-09-12 12:28:12 +0000404 ret['reviewers'] = set(r['author'] for r in ret['replies'])
405 ret['reviewers'].discard(ret['author'])
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +0000406 ret['bug'] = self.extract_bug_number_from_description(issue)
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000407 return ret
408
409 @staticmethod
deymo@chromium.org6c039202013-09-12 12:28:12 +0000410 def process_gerrit_ssh_issue_replies(replies):
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000411 ret = []
412 replies = filter(lambda r: 'email' in r['reviewer'], replies)
413 for reply in replies:
deymo@chromium.org6c039202013-09-12 12:28:12 +0000414 ret.append({
415 'author': reply['reviewer']['email'],
416 'created': datetime.fromtimestamp(reply['timestamp']),
417 'content': '',
418 })
419 return ret
420
421 def process_gerrit_rest_issue(self, instance, issue):
422 ret = {}
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000423 if self.options.deltas:
424 ret['delta'] = DefaultFormatter().format(
425 '+{insertions},-{deletions}',
426 **issue)
427 ret['status'] = issue['status']
deymo@chromium.org6c039202013-09-12 12:28:12 +0000428 ret['review_url'] = 'https://%s/%s' % (instance['url'], issue['_number'])
429 if 'shorturl' in instance:
430 # TODO(deymo): Move this short link to https once crosreview.com supports
431 # it.
432 ret['review_url'] = 'http://%s/%s' % (instance['shorturl'],
433 issue['_number'])
434 ret['header'] = issue['subject']
435 ret['owner'] = issue['owner']['email']
436 ret['author'] = ret['owner']
437 ret['created'] = datetime_from_gerrit(issue['created'])
438 ret['modified'] = datetime_from_gerrit(issue['updated'])
439 if 'messages' in issue:
440 ret['replies'] = self.process_gerrit_rest_issue_replies(issue['messages'])
441 else:
442 ret['replies'] = []
443 ret['reviewers'] = set(r['author'] for r in ret['replies'])
444 ret['reviewers'].discard(ret['author'])
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +0000445 ret['bug'] = self.extract_bug_number_from_description(issue)
deymo@chromium.org6c039202013-09-12 12:28:12 +0000446 return ret
447
448 @staticmethod
449 def process_gerrit_rest_issue_replies(replies):
450 ret = []
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000451 replies = filter(lambda r: 'author' in r and 'email' in r['author'],
452 replies)
deymo@chromium.org6c039202013-09-12 12:28:12 +0000453 for reply in replies:
454 ret.append({
455 'author': reply['author']['email'],
456 'created': datetime_from_gerrit(reply['date']),
457 'content': reply['message'],
458 })
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000459 return ret
460
seanmccullough@chromium.org3e4a5812015-06-11 17:48:47 +0000461 def project_hosting_issue_search(self, instance):
462 auth_config = auth.extract_auth_config_from_options(self.options)
463 authenticator = auth.get_authenticator_for_host(
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000464 'bugs.chromium.org', auth_config)
seanmccullough@chromium.org3e4a5812015-06-11 17:48:47 +0000465 http = authenticator.authorize(httplib2.Http())
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000466 url = ('https://monorail-prod.appspot.com/_ah/api/monorail/v1/projects'
467 '/%s/issues') % instance['name']
seanmccullough@chromium.org3e4a5812015-06-11 17:48:47 +0000468 epoch = datetime.utcfromtimestamp(0)
469 user_str = '%s@chromium.org' % self.user
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000470
seanmccullough@chromium.org3e4a5812015-06-11 17:48:47 +0000471 query_data = urllib.urlencode({
472 'maxResults': 10000,
473 'q': user_str,
474 'publishedMax': '%d' % (self.modified_before - epoch).total_seconds(),
475 'updatedMin': '%d' % (self.modified_after - epoch).total_seconds(),
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000476 })
seanmccullough@chromium.org3e4a5812015-06-11 17:48:47 +0000477 url = url + '?' + query_data
478 _, body = http.request(url)
479 content = json.loads(body)
480 if not content:
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000481 logging.error('Unable to parse %s response from projecthosting.',
482 instance['name'])
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000483 return []
484
seanmccullough@chromium.org3e4a5812015-06-11 17:48:47 +0000485 issues = []
486 if 'items' in content:
487 items = content['items']
488 for item in items:
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +0000489 if instance.get('shorturl'):
490 item_url = 'https://%s/%d' % (instance['shorturl'], item['id'])
491 else:
492 item_url = 'https://bugs.chromium.org/p/%s/issues/detail?id=%d' % (
493 instance['name'], item['id'])
seanmccullough@chromium.org3e4a5812015-06-11 17:48:47 +0000494 issue = {
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000495 'header': item['title'],
496 'created': dateutil.parser.parse(item['published']),
497 'modified': dateutil.parser.parse(item['updated']),
498 'author': item['author']['name'],
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +0000499 'url': item_url,
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000500 'comments': [],
501 'status': item['status'],
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +0000502 'labels': [],
503 'components': []
seanmccullough@chromium.org3e4a5812015-06-11 17:48:47 +0000504 }
jdoerrie64287842017-01-09 14:40:26 +0100505 if 'shorturl' in instance:
506 issue['url'] = 'http://%s/%d' % (instance['shorturl'], item['id'])
507
seanmccullough@chromium.org3e4a5812015-06-11 17:48:47 +0000508 if 'owner' in item:
509 issue['owner'] = item['owner']['name']
510 else:
511 issue['owner'] = 'None'
512 if issue['owner'] == user_str or issue['author'] == user_str:
513 issues.append(issue)
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +0000514 if 'labels' in item:
515 issue['labels'] = item['labels']
516 if 'components' in item:
517 issue['components'] = item['components']
seanmccullough@chromium.org3e4a5812015-06-11 17:48:47 +0000518
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000519 return issues
520
jsbell@chromium.orgc92f5822014-01-06 23:49:11 +0000521 def print_heading(self, heading):
522 print
523 print self.options.output_format_heading.format(heading=heading)
524
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000525 def match(self, author):
526 if '@' in self.user:
527 return author == self.user
528 return author.startswith(self.user + '@')
529
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000530 def print_change(self, change):
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000531 activity = len([
532 reply
533 for reply in change['replies']
534 if self.match(reply['author'])
535 ])
cjhopman@chromium.org53c1e562013-03-11 20:02:38 +0000536 optional_values = {
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000537 'created': change['created'].date().isoformat(),
538 'modified': change['modified'].date().isoformat(),
539 'reviewers': ', '.join(change['reviewers']),
540 'status': change['status'],
541 'activity': activity,
cjhopman@chromium.org53c1e562013-03-11 20:02:38 +0000542 }
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000543 if self.options.deltas:
544 optional_values['delta'] = change['delta']
545
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000546 self.print_generic(self.options.output_format,
547 self.options.output_format_changes,
548 change['header'],
549 change['review_url'],
cjhopman@chromium.org53c1e562013-03-11 20:02:38 +0000550 change['author'],
551 optional_values)
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000552
553 def print_issue(self, issue):
554 optional_values = {
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000555 'created': issue['created'].date().isoformat(),
556 'modified': issue['modified'].date().isoformat(),
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000557 'owner': issue['owner'],
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000558 'status': issue['status'],
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000559 }
560 self.print_generic(self.options.output_format,
561 self.options.output_format_issues,
562 issue['header'],
563 issue['url'],
564 issue['author'],
565 optional_values)
566
567 def print_review(self, review):
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000568 activity = len([
569 reply
570 for reply in review['replies']
571 if self.match(reply['author'])
572 ])
573 optional_values = {
574 'created': review['created'].date().isoformat(),
575 'modified': review['modified'].date().isoformat(),
576 'activity': activity,
577 }
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000578 self.print_generic(self.options.output_format,
579 self.options.output_format_reviews,
580 review['header'],
581 review['review_url'],
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000582 review['author'],
583 optional_values)
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000584
585 @staticmethod
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000586 def print_generic(default_fmt, specific_fmt,
587 title, url, author,
588 optional_values=None):
589 output_format = specific_fmt if specific_fmt is not None else default_fmt
590 output_format = unicode(output_format)
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000591 values = {
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000592 'title': title,
593 'url': url,
594 'author': author,
595 }
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000596 if optional_values is not None:
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000597 values.update(optional_values)
598 print DefaultFormatter().format(output_format, **values).encode(
599 sys.getdefaultencoding())
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000600
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000601
602 def filter_issue(self, issue, should_filter_by_user=True):
603 def maybe_filter_username(email):
604 return not should_filter_by_user or username(email) == self.user
605 if (maybe_filter_username(issue['author']) and
606 self.filter_modified(issue['created'])):
607 return True
608 if (maybe_filter_username(issue['owner']) and
609 (self.filter_modified(issue['created']) or
610 self.filter_modified(issue['modified']))):
611 return True
612 for reply in issue['replies']:
613 if self.filter_modified(reply['created']):
614 if not should_filter_by_user:
615 break
616 if (username(reply['author']) == self.user
617 or (self.user + '@') in reply['content']):
618 break
619 else:
620 return False
621 return True
622
623 def filter_modified(self, modified):
624 return self.modified_after < modified and modified < self.modified_before
625
626 def auth_for_changes(self):
627 #TODO(cjhopman): Move authentication check for getting changes here.
628 pass
629
630 def auth_for_reviews(self):
631 # Reviews use all the same instances as changes so no authentication is
632 # required.
633 pass
634
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000635 def get_changes(self):
636 for instance in rietveld_instances:
637 self.changes += self.rietveld_search(instance, owner=self.user)
638
639 for instance in gerrit_instances:
640 self.changes += self.gerrit_search(instance, owner=self.user)
641
642 def print_changes(self):
643 if self.changes:
jsbell@chromium.orgc92f5822014-01-06 23:49:11 +0000644 self.print_heading('Changes')
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000645 for change in self.changes:
646 self.print_change(change)
647
648 def get_reviews(self):
649 for instance in rietveld_instances:
650 self.reviews += self.rietveld_search(instance, reviewer=self.user)
651
652 for instance in gerrit_instances:
653 reviews = self.gerrit_search(instance, reviewer=self.user)
654 reviews = filter(lambda r: not username(r['owner']) == self.user, reviews)
655 self.reviews += reviews
656
657 def print_reviews(self):
658 if self.reviews:
jsbell@chromium.orgc92f5822014-01-06 23:49:11 +0000659 self.print_heading('Reviews')
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000660 for review in self.reviews:
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000661 self.print_review(review)
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000662
663 def get_issues(self):
664 for project in google_code_projects:
seanmccullough@chromium.org3e4a5812015-06-11 17:48:47 +0000665 self.issues += self.project_hosting_issue_search(project)
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000666
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000667 def print_issues(self):
668 if self.issues:
jsbell@chromium.orgc92f5822014-01-06 23:49:11 +0000669 self.print_heading('Issues')
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000670 for issue in self.issues:
671 self.print_issue(issue)
672
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000673 def print_activity(self):
674 self.print_changes()
675 self.print_reviews()
676 self.print_issues()
677
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +0000678 def dump_json(self, ignore_keys=None):
679 if ignore_keys is None:
680 ignore_keys = ['replies']
681
682 def format_for_json_dump(in_array):
683 output = {}
684 for item in in_array:
685 url = item.get('url') or item.get('review_url')
686 if not url:
687 raise Exception('Dumped item %s does not specify url' % item)
688 output[url] = dict(
689 (k, v) for k,v in item.iteritems() if k not in ignore_keys)
690 return output
691
692 class PythonObjectEncoder(json.JSONEncoder):
693 def default(self, obj): # pylint: disable=method-hidden
694 if isinstance(obj, datetime):
695 return obj.isoformat()
696 if isinstance(obj, set):
697 return list(obj)
698 return json.JSONEncoder.default(self, obj)
699
700 output = {
701 'reviews': format_for_json_dump(self.reviews),
702 'changes': format_for_json_dump(self.changes),
703 'issues': format_for_json_dump(self.issues)
704 }
705 print json.dumps(output, indent=2, cls=PythonObjectEncoder)
706
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000707
708def main():
709 # Silence upload.py.
710 rietveld.upload.verbosity = 0
711
712 parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
713 parser.add_option(
714 '-u', '--user', metavar='<email>',
715 default=os.environ.get('USER'),
716 help='Filter on user, default=%default')
717 parser.add_option(
718 '-b', '--begin', metavar='<date>',
wychen@chromium.org85cab632015-05-28 21:04:37 +0000719 help='Filter issues created after the date (mm/dd/yy)')
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000720 parser.add_option(
721 '-e', '--end', metavar='<date>',
wychen@chromium.org85cab632015-05-28 21:04:37 +0000722 help='Filter issues created before the date (mm/dd/yy)')
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000723 quarter_begin, quarter_end = get_quarter_of(datetime.today() -
724 relativedelta(months=2))
725 parser.add_option(
726 '-Q', '--last_quarter', action='store_true',
jsbell@chromium.org74bfde02014-04-09 17:14:54 +0000727 help='Use last quarter\'s dates, i.e. %s to %s' % (
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000728 quarter_begin.strftime('%Y-%m-%d'), quarter_end.strftime('%Y-%m-%d')))
729 parser.add_option(
730 '-Y', '--this_year', action='store_true',
731 help='Use this year\'s dates')
732 parser.add_option(
733 '-w', '--week_of', metavar='<date>',
wychen@chromium.org85cab632015-05-28 21:04:37 +0000734 help='Show issues for week of the date (mm/dd/yy)')
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000735 parser.add_option(
wychen@chromium.org8ba1ddb2015-04-29 00:04:25 +0000736 '-W', '--last_week', action='count',
737 help='Show last week\'s issues. Use more times for more weeks.')
jsbell@chromium.org74bfde02014-04-09 17:14:54 +0000738 parser.add_option(
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000739 '-a', '--auth',
740 action='store_true',
741 help='Ask to authenticate for instances with no auth cookie')
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000742 parser.add_option(
743 '-d', '--deltas',
744 action='store_true',
745 help='Fetch deltas for changes (slow).')
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000746
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000747 activity_types_group = optparse.OptionGroup(parser, 'Activity Types',
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000748 'By default, all activity will be looked up and '
749 'printed. If any of these are specified, only '
750 'those specified will be searched.')
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000751 activity_types_group.add_option(
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000752 '-c', '--changes',
753 action='store_true',
754 help='Show changes.')
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000755 activity_types_group.add_option(
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000756 '-i', '--issues',
757 action='store_true',
758 help='Show issues.')
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000759 activity_types_group.add_option(
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000760 '-r', '--reviews',
761 action='store_true',
762 help='Show reviews.')
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000763 parser.add_option_group(activity_types_group)
764
765 output_format_group = optparse.OptionGroup(parser, 'Output Format',
766 'By default, all activity will be printed in the '
767 'following format: {url} {title}. This can be '
768 'changed for either all activity types or '
769 'individually for each activity type. The format '
770 'is defined as documented for '
771 'string.format(...). The variables available for '
772 'all activity types are url, title and author. '
773 'Format options for specific activity types will '
774 'override the generic format.')
775 output_format_group.add_option(
776 '-f', '--output-format', metavar='<format>',
777 default=u'{url} {title}',
778 help='Specifies the format to use when printing all your activity.')
779 output_format_group.add_option(
780 '--output-format-changes', metavar='<format>',
781 default=None,
cjhopman@chromium.org53c1e562013-03-11 20:02:38 +0000782 help='Specifies the format to use when printing changes. Supports the '
783 'additional variable {reviewers}')
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000784 output_format_group.add_option(
785 '--output-format-issues', metavar='<format>',
786 default=None,
cjhopman@chromium.org53c1e562013-03-11 20:02:38 +0000787 help='Specifies the format to use when printing issues. Supports the '
788 'additional variable {owner}.')
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000789 output_format_group.add_option(
790 '--output-format-reviews', metavar='<format>',
791 default=None,
792 help='Specifies the format to use when printing reviews.')
jsbell@chromium.orgc92f5822014-01-06 23:49:11 +0000793 output_format_group.add_option(
794 '--output-format-heading', metavar='<format>',
795 default=u'{heading}:',
796 help='Specifies the format to use when printing headings.')
797 output_format_group.add_option(
798 '-m', '--markdown', action='store_true',
799 help='Use markdown-friendly output (overrides --output-format '
800 'and --output-format-heading)')
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +0000801 output_format_group.add_option(
802 '-j', '--json', action='store_true',
803 help='Output json data (overrides other format options)')
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000804 parser.add_option_group(output_format_group)
vadimsh@chromium.orgcf6a5d22015-04-09 22:02:00 +0000805 auth.add_auth_options(parser)
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000806
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000807 parser.add_option(
808 '-v', '--verbose',
809 action='store_const',
810 dest='verbosity',
811 default=logging.WARN,
812 const=logging.INFO,
813 help='Output extra informational messages.'
814 )
815 parser.add_option(
816 '-q', '--quiet',
817 action='store_const',
818 dest='verbosity',
819 const=logging.ERROR,
820 help='Suppress non-error messages.'
821 )
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +0000822 parser.add_option(
823 '-o', '--output', metavar='<file>',
824 help='Where to output the results. By default prints to stdout.')
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000825
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000826 # Remove description formatting
827 parser.format_description = (
Quinten Yearsleyb2cc4a92016-12-15 13:53:26 -0800828 lambda _: parser.description) # pylint: disable=no-member
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000829
830 options, args = parser.parse_args()
831 options.local_user = os.environ.get('USER')
832 if args:
833 parser.error('Args unsupported')
834 if not options.user:
835 parser.error('USER is not set, please use -u')
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000836 options.user = username(options.user)
837
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000838 logging.basicConfig(level=options.verbosity)
839
840 # python-keyring provides easy access to the system keyring.
841 try:
842 import keyring # pylint: disable=unused-import,unused-variable,F0401
843 except ImportError:
844 logging.warning('Consider installing python-keyring')
845
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000846 if not options.begin:
847 if options.last_quarter:
848 begin, end = quarter_begin, quarter_end
849 elif options.this_year:
850 begin, end = get_year_of(datetime.today())
851 elif options.week_of:
852 begin, end = (get_week_of(datetime.strptime(options.week_of, '%m/%d/%y')))
jsbell@chromium.org74bfde02014-04-09 17:14:54 +0000853 elif options.last_week:
wychen@chromium.org8ba1ddb2015-04-29 00:04:25 +0000854 begin, end = (get_week_of(datetime.today() -
855 timedelta(days=1 + 7 * options.last_week)))
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000856 else:
857 begin, end = (get_week_of(datetime.today() - timedelta(days=1)))
858 else:
859 begin = datetime.strptime(options.begin, '%m/%d/%y')
860 if options.end:
861 end = datetime.strptime(options.end, '%m/%d/%y')
862 else:
863 end = datetime.today()
864 options.begin, options.end = begin, end
865
jsbell@chromium.orgc92f5822014-01-06 23:49:11 +0000866 if options.markdown:
867 options.output_format = ' * [{title}]({url})'
868 options.output_format_heading = '### {heading} ###'
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000869 logging.info('Searching for activity by %s', options.user)
870 logging.info('Using range %s to %s', options.begin, options.end)
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000871
872 my_activity = MyActivity(options)
873
874 if not (options.changes or options.reviews or options.issues):
875 options.changes = True
876 options.issues = True
877 options.reviews = True
878
879 # First do any required authentication so none of the user interaction has to
880 # wait for actual work.
881 if options.changes:
882 my_activity.auth_for_changes()
883 if options.reviews:
884 my_activity.auth_for_reviews()
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000885
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000886 logging.info('Looking up activity.....')
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000887
seanmccullough@chromium.org3e4a5812015-06-11 17:48:47 +0000888 try:
889 if options.changes:
890 my_activity.get_changes()
891 if options.reviews:
892 my_activity.get_reviews()
893 if options.issues:
894 my_activity.get_issues()
895 except auth.AuthenticationError as e:
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000896 logging.error('auth.AuthenticationError: %s', e)
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000897
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +0000898 output_file = None
899 try:
900 if options.output:
901 output_file = open(options.output, 'w')
902 logging.info('Printing output to "%s"', options.output)
903 sys.stdout = output_file
904 except (IOError, OSError) as e:
905 logging.error('Unable to write output: %s', e)
906 else:
907 if options.json:
908 my_activity.dump_json()
909 else:
910 my_activity.print_changes()
911 my_activity.print_reviews()
912 my_activity.print_issues()
913 finally:
914 if output_file:
915 logging.info('Done printing to file.')
916 sys.stdout = sys.__stdout__
917 output_file.close()
918
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000919 return 0
920
921
922if __name__ == '__main__':
stevefung@chromium.org832d51e2015-05-27 12:52:51 +0000923 # Fix encoding to support non-ascii issue titles.
924 fix_encoding.fix_encoding()
925
sbc@chromium.org013731e2015-02-26 18:28:43 +0000926 try:
927 sys.exit(main())
928 except KeyboardInterrupt:
929 sys.stderr.write('interrupted\n')
930 sys.exit(1)