blob: 26817f36fff413fd9dbe22bab77a0ce2b50ecc0b [file] [log] [blame]
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +00001#!/usr/bin/env python
2# Copyright (c) 2012 The Chromium Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Get stats about your activity.
7
8Example:
9 - my_activity.py for stats for the current week (last week on mondays).
10 - my_activity.py -Q for stats for last quarter.
11 - my_activity.py -Y for stats for this year.
12 - my_activity.py -b 4/5/12 for stats since 4/5/12.
13 - my_activity.py -b 4/5/12 -e 6/7/12 for stats between 4/5/12 and 6/7/12.
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +000014 - my_activity.py -jd to output stats for the week to json with deltas data.
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +000015"""
16
vadimsh@chromium.orgcf6a5d22015-04-09 22:02:00 +000017# TODO(vadimsh): This script knows too much about ClientLogin and cookies. It
18# will stop to work on ~20 Apr 2015.
19
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +000020# These services typically only provide a created time and a last modified time
21# for each item for general queries. This is not enough to determine if there
22# was activity in a given time period. So, we first query for all things created
23# before end and modified after begin. Then, we get the details of each item and
24# check those details to determine if there was activity in the given period.
25# This means that query time scales mostly with (today() - begin).
26
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +000027from datetime import datetime
28from datetime import timedelta
29from functools import partial
30import json
Tobias Sargeantffb3c432017-03-08 14:09:14 +000031import logging
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +000032import optparse
33import os
34import subprocess
Tobias Sargeantffb3c432017-03-08 14:09:14 +000035from string import Formatter
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +000036import sys
37import urllib
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +000038import re
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +000039
vadimsh@chromium.orgcf6a5d22015-04-09 22:02:00 +000040import auth
stevefung@chromium.org832d51e2015-05-27 12:52:51 +000041import fix_encoding
deymo@chromium.orgf8be2762013-11-06 01:01:59 +000042import gerrit_util
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +000043import rietveld
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +000044
seanmccullough@chromium.org3e4a5812015-06-11 17:48:47 +000045from third_party import httplib2
46
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +000047try:
Tobias Sargeantffb3c432017-03-08 14:09:14 +000048 import dateutil # pylint: disable=import-error
49 import dateutil.parser
50 from dateutil.relativedelta import relativedelta
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +000051except ImportError:
Tobias Sargeantffb3c432017-03-08 14:09:14 +000052 logging.error('python-dateutil package required')
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +000053 exit(1)
54
Tobias Sargeantffb3c432017-03-08 14:09:14 +000055
56class DefaultFormatter(Formatter):
57 def __init__(self, default = ''):
58 super(DefaultFormatter, self).__init__()
59 self.default = default
60
61 def get_value(self, key, args, kwds):
62 if isinstance(key, basestring) and key not in kwds:
63 return self.default
64 return Formatter.get_value(self, key, args, kwds)
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +000065
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +000066rietveld_instances = [
67 {
68 'url': 'codereview.chromium.org',
69 'shorturl': 'crrev.com',
70 'supports_owner_modified_query': True,
71 'requires_auth': False,
72 'email_domain': 'chromium.org',
Varun Khanejad9f97bc2017-08-02 12:55:01 -070073 'short_url_protocol': 'https',
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +000074 },
75 {
76 'url': 'chromereviews.googleplex.com',
77 'shorturl': 'go/chromerev',
78 'supports_owner_modified_query': True,
79 'requires_auth': True,
80 'email_domain': 'google.com',
81 },
82 {
83 'url': 'codereview.appspot.com',
84 'supports_owner_modified_query': True,
85 'requires_auth': False,
86 'email_domain': 'chromium.org',
87 },
88 {
89 'url': 'breakpad.appspot.com',
90 'supports_owner_modified_query': False,
91 'requires_auth': False,
92 'email_domain': 'chromium.org',
93 },
94]
95
96gerrit_instances = [
97 {
deymo@chromium.org6c039202013-09-12 12:28:12 +000098 'url': 'chromium-review.googlesource.com',
Jeremy Romanf475a472017-06-06 16:49:11 -040099 'shorturl': 'crrev.com/c',
Varun Khanejad9f97bc2017-08-02 12:55:01 -0700100 'short_url_protocol': 'https',
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000101 },
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000102 {
103 'url': 'chrome-internal-review.googlesource.com',
Jeremy Romanf475a472017-06-06 16:49:11 -0400104 'shorturl': 'crrev.com/i',
Varun Khanejad9f97bc2017-08-02 12:55:01 -0700105 'short_url_protocol': 'https',
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000106 },
deymo@chromium.org56dc57a2015-09-10 18:26:54 +0000107 {
108 'url': 'android-review.googlesource.com',
109 },
Ryan Harrison897602a2017-09-18 16:23:41 -0400110 {
111 'url': 'pdfium-review.googlesource.com',
112 },
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000113]
114
115google_code_projects = [
116 {
117 'name': 'chromium',
118 'shorturl': 'crbug.com',
Varun Khanejad9f97bc2017-08-02 12:55:01 -0700119 'short_url_protocol': 'https',
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000120 },
121 {
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000122 'name': 'google-breakpad',
123 },
124 {
125 'name': 'gyp',
enne@chromium.orgf01fad32012-11-26 18:09:38 +0000126 },
127 {
128 'name': 'skia',
129 },
Ryan Harrison897602a2017-09-18 16:23:41 -0400130 {
131 'name': 'pdfium',
132 'shorturl': 'crbug.com/pdfium',
133 'short_url_protocol': 'https',
134 },
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000135]
136
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000137def username(email):
138 """Keeps the username of an email address."""
139 return email and email.split('@', 1)[0]
140
141
cjhopman@chromium.org426557a2012-10-22 20:18:52 +0000142def datetime_to_midnight(date):
143 return date - timedelta(hours=date.hour, minutes=date.minute,
144 seconds=date.second, microseconds=date.microsecond)
145
146
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000147def get_quarter_of(date):
cjhopman@chromium.org426557a2012-10-22 20:18:52 +0000148 begin = (datetime_to_midnight(date) -
149 relativedelta(months=(date.month % 3) - 1, days=(date.day - 1)))
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000150 return begin, begin + relativedelta(months=3)
151
152
153def get_year_of(date):
cjhopman@chromium.org426557a2012-10-22 20:18:52 +0000154 begin = (datetime_to_midnight(date) -
155 relativedelta(months=(date.month - 1), days=(date.day - 1)))
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000156 return begin, begin + relativedelta(years=1)
157
158
159def get_week_of(date):
cjhopman@chromium.org426557a2012-10-22 20:18:52 +0000160 begin = (datetime_to_midnight(date) - timedelta(days=date.weekday()))
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000161 return begin, begin + timedelta(days=7)
162
163
164def get_yes_or_no(msg):
165 while True:
166 response = raw_input(msg + ' yes/no [no] ')
167 if response == 'y' or response == 'yes':
168 return True
169 elif not response or response == 'n' or response == 'no':
170 return False
171
172
deymo@chromium.org6c039202013-09-12 12:28:12 +0000173def datetime_from_gerrit(date_string):
174 return datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S.%f000')
175
176
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000177def datetime_from_rietveld(date_string):
deymo@chromium.org29eb6e62014-03-20 01:55:55 +0000178 try:
179 return datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S.%f')
180 except ValueError:
181 # Sometimes rietveld returns a value without the milliseconds part, so we
182 # attempt to parse those cases as well.
183 return datetime.strptime(date_string, '%Y-%m-%d %H:%M:%S')
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000184
185
186def datetime_from_google_code(date_string):
187 return datetime.strptime(date_string, '%Y-%m-%dT%H:%M:%S.%fZ')
188
189
190class MyActivity(object):
191 def __init__(self, options):
192 self.options = options
193 self.modified_after = options.begin
194 self.modified_before = options.end
195 self.user = options.user
196 self.changes = []
197 self.reviews = []
198 self.issues = []
199 self.check_cookies()
200 self.google_code_auth_token = None
201
202 # Check the codereview cookie jar to determine which Rietveld instances to
203 # authenticate to.
204 def check_cookies(self):
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000205 filtered_instances = []
206
207 def has_cookie(instance):
seanmccullough@chromium.org3e4a5812015-06-11 17:48:47 +0000208 auth_config = auth.extract_auth_config_from_options(self.options)
209 a = auth.get_authenticator_for_host(instance['url'], auth_config)
210 return a.has_cached_credentials()
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000211
212 for instance in rietveld_instances:
213 instance['auth'] = has_cookie(instance)
214
215 if filtered_instances:
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000216 logging.warning('No cookie found for the following Rietveld instance%s:',
217 's' if len(filtered_instances) > 1 else '')
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000218 for instance in filtered_instances:
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000219 logging.warning('\t' + instance['url'])
220 logging.warning('Use --auth if you would like to authenticate to them.')
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000221
222 def rietveld_search(self, instance, owner=None, reviewer=None):
223 if instance['requires_auth'] and not instance['auth']:
224 return []
225
226
227 email = None if instance['auth'] else ''
vadimsh@chromium.orgcf6a5d22015-04-09 22:02:00 +0000228 auth_config = auth.extract_auth_config_from_options(self.options)
229 remote = rietveld.Rietveld('https://' + instance['url'], auth_config, email)
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000230
231 # See def search() in rietveld.py to see all the filters you can use.
232 query_modified_after = None
233
234 if instance['supports_owner_modified_query']:
235 query_modified_after = self.modified_after.strftime('%Y-%m-%d')
236
237 # Rietveld does not allow search by both created_before and modified_after.
238 # (And some instances don't allow search by both owner and modified_after)
239 owner_email = None
240 reviewer_email = None
241 if owner:
242 owner_email = owner + '@' + instance['email_domain']
243 if reviewer:
244 reviewer_email = reviewer + '@' + instance['email_domain']
245 issues = remote.search(
246 owner=owner_email,
247 reviewer=reviewer_email,
248 modified_after=query_modified_after,
249 with_messages=True)
250
251 issues = filter(
252 lambda i: (datetime_from_rietveld(i['created']) < self.modified_before),
253 issues)
254 issues = filter(
255 lambda i: (datetime_from_rietveld(i['modified']) > self.modified_after),
256 issues)
257
258 should_filter_by_user = True
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000259 issues = map(partial(self.process_rietveld_issue, remote, instance), issues)
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000260 issues = filter(
261 partial(self.filter_issue, should_filter_by_user=should_filter_by_user),
262 issues)
263 issues = sorted(issues, key=lambda i: i['modified'], reverse=True)
264
265 return issues
266
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +0000267 def extract_bug_number_from_description(self, issue):
268 description = None
269
270 if 'description' in issue:
271 # Getting the description for Rietveld
272 description = issue['description']
273 elif 'revisions' in issue:
274 # Getting the description for REST Gerrit
275 revision = issue['revisions'][issue['current_revision']]
276 description = revision['commit']['message']
277
278 bugs = []
279 if description:
Nicolas Dossou-gbete903ea732017-07-10 16:46:59 +0100280 # Handle both "Bug: 99999" and "BUG=99999" bug notations
281 # Multiple bugs can be noted on a single line or in multiple ones.
282 matches = re.findall(r'BUG[=:]\s?(((\d+)(,\s?)?)+)', description,
283 flags=re.IGNORECASE)
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +0000284 if matches:
285 for match in matches:
286 bugs.extend(match[0].replace(' ', '').split(','))
287
288 return bugs
289
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000290 def process_rietveld_issue(self, remote, instance, issue):
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000291 ret = {}
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000292 if self.options.deltas:
293 patchset_props = remote.get_patchset_properties(
294 issue['issue'],
295 issue['patchsets'][-1])
296 ret['delta'] = '+%d,-%d' % (
297 sum(f['num_added'] for f in patchset_props['files'].itervalues()),
298 sum(f['num_removed'] for f in patchset_props['files'].itervalues()))
299
300 if issue['landed_days_ago'] != 'unknown':
301 ret['status'] = 'committed'
302 elif issue['closed']:
303 ret['status'] = 'closed'
304 elif len(issue['reviewers']) and issue['all_required_reviewers_approved']:
305 ret['status'] = 'ready'
306 else:
307 ret['status'] = 'open'
308
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000309 ret['owner'] = issue['owner_email']
310 ret['author'] = ret['owner']
311
cjhopman@chromium.org53c1e562013-03-11 20:02:38 +0000312 ret['reviewers'] = set(issue['reviewers'])
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000313
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000314 if 'shorturl' in instance:
Varun Khanejad9f97bc2017-08-02 12:55:01 -0700315 url = instance['shorturl']
316 protocol = instance.get('short_url_protocol', 'http')
317 else:
318 url = instance['url']
319 protocol = 'https'
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000320
Varun Khanejad9f97bc2017-08-02 12:55:01 -0700321 ret['review_url'] = '%s://%s/%d' % (protocol, url, issue['issue'])
cjhopman@chromium.org53c1e562013-03-11 20:02:38 +0000322
323 # Rietveld sometimes has '\r\n' instead of '\n'.
324 ret['header'] = issue['description'].replace('\r', '').split('\n')[0]
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000325
326 ret['modified'] = datetime_from_rietveld(issue['modified'])
327 ret['created'] = datetime_from_rietveld(issue['created'])
328 ret['replies'] = self.process_rietveld_replies(issue['messages'])
329
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +0000330 ret['bug'] = self.extract_bug_number_from_description(issue)
331 ret['landed_days_ago'] = issue['landed_days_ago']
332
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000333 return ret
334
335 @staticmethod
336 def process_rietveld_replies(replies):
337 ret = []
338 for reply in replies:
339 r = {}
340 r['author'] = reply['sender']
341 r['created'] = datetime_from_rietveld(reply['date'])
342 r['content'] = ''
343 ret.append(r)
344 return ret
345
deymo@chromium.org6c039202013-09-12 12:28:12 +0000346 @staticmethod
deymo@chromium.org6c039202013-09-12 12:28:12 +0000347 def gerrit_changes_over_rest(instance, filters):
Michael Achenbach6fbf12f2017-07-06 10:54:11 +0200348 # Convert the "key:value" filter to a list of (key, value) pairs.
349 req = list(f.split(':', 1) for f in filters)
deymo@chromium.org6c039202013-09-12 12:28:12 +0000350 try:
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000351 # Instantiate the generator to force all the requests now and catch the
352 # errors here.
353 return list(gerrit_util.GenerateAllChanges(instance['url'], req,
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +0000354 o_params=['MESSAGES', 'LABELS', 'DETAILED_ACCOUNTS',
355 'CURRENT_REVISION', 'CURRENT_COMMIT']))
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000356 except gerrit_util.GerritError, e:
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000357 logging.error('Looking up %r: %s', instance['url'], e)
deymo@chromium.org6c039202013-09-12 12:28:12 +0000358 return []
359
deymo@chromium.org6c039202013-09-12 12:28:12 +0000360 def gerrit_search(self, instance, owner=None, reviewer=None):
361 max_age = datetime.today() - self.modified_after
362 max_age = max_age.days * 24 * 3600 + max_age.seconds
363 user_filter = 'owner:%s' % owner if owner else 'reviewer:%s' % reviewer
364 filters = ['-age:%ss' % max_age, user_filter]
365
Aaron Gable2979a872017-09-05 17:38:32 -0700366 issues = self.gerrit_changes_over_rest(instance, filters)
367 issues = [self.process_gerrit_issue(instance, issue)
368 for issue in issues]
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000369
370 # TODO(cjhopman): should we filter abandoned changes?
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000371 issues = filter(self.filter_issue, issues)
372 issues = sorted(issues, key=lambda i: i['modified'], reverse=True)
373
374 return issues
375
Aaron Gable2979a872017-09-05 17:38:32 -0700376 def process_gerrit_issue(self, instance, issue):
deymo@chromium.org6c039202013-09-12 12:28:12 +0000377 ret = {}
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000378 if self.options.deltas:
379 ret['delta'] = DefaultFormatter().format(
380 '+{insertions},-{deletions}',
381 **issue)
382 ret['status'] = issue['status']
deymo@chromium.org6c039202013-09-12 12:28:12 +0000383 if 'shorturl' in instance:
Varun Khanejad9f97bc2017-08-02 12:55:01 -0700384 protocol = instance.get('short_url_protocol', 'http')
385 url = instance['shorturl']
386 else:
387 protocol = 'https'
388 url = instance['url']
389 ret['review_url'] = '%s://%s/%s' % (protocol, url, issue['_number'])
390
deymo@chromium.org6c039202013-09-12 12:28:12 +0000391 ret['header'] = issue['subject']
392 ret['owner'] = issue['owner']['email']
393 ret['author'] = ret['owner']
394 ret['created'] = datetime_from_gerrit(issue['created'])
395 ret['modified'] = datetime_from_gerrit(issue['updated'])
396 if 'messages' in issue:
Aaron Gable2979a872017-09-05 17:38:32 -0700397 ret['replies'] = self.process_gerrit_issue_replies(issue['messages'])
deymo@chromium.org6c039202013-09-12 12:28:12 +0000398 else:
399 ret['replies'] = []
400 ret['reviewers'] = set(r['author'] for r in ret['replies'])
401 ret['reviewers'].discard(ret['author'])
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +0000402 ret['bug'] = self.extract_bug_number_from_description(issue)
deymo@chromium.org6c039202013-09-12 12:28:12 +0000403 return ret
404
405 @staticmethod
Aaron Gable2979a872017-09-05 17:38:32 -0700406 def process_gerrit_issue_replies(replies):
deymo@chromium.org6c039202013-09-12 12:28:12 +0000407 ret = []
deymo@chromium.orgf8be2762013-11-06 01:01:59 +0000408 replies = filter(lambda r: 'author' in r and 'email' in r['author'],
409 replies)
deymo@chromium.org6c039202013-09-12 12:28:12 +0000410 for reply in replies:
411 ret.append({
412 'author': reply['author']['email'],
413 'created': datetime_from_gerrit(reply['date']),
414 'content': reply['message'],
415 })
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000416 return ret
417
seanmccullough@chromium.org3e4a5812015-06-11 17:48:47 +0000418 def project_hosting_issue_search(self, instance):
419 auth_config = auth.extract_auth_config_from_options(self.options)
420 authenticator = auth.get_authenticator_for_host(
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000421 'bugs.chromium.org', auth_config)
seanmccullough@chromium.org3e4a5812015-06-11 17:48:47 +0000422 http = authenticator.authorize(httplib2.Http())
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000423 url = ('https://monorail-prod.appspot.com/_ah/api/monorail/v1/projects'
424 '/%s/issues') % instance['name']
seanmccullough@chromium.org3e4a5812015-06-11 17:48:47 +0000425 epoch = datetime.utcfromtimestamp(0)
426 user_str = '%s@chromium.org' % self.user
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000427
seanmccullough@chromium.org3e4a5812015-06-11 17:48:47 +0000428 query_data = urllib.urlencode({
429 'maxResults': 10000,
430 'q': user_str,
431 'publishedMax': '%d' % (self.modified_before - epoch).total_seconds(),
432 'updatedMin': '%d' % (self.modified_after - epoch).total_seconds(),
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000433 })
seanmccullough@chromium.org3e4a5812015-06-11 17:48:47 +0000434 url = url + '?' + query_data
435 _, body = http.request(url)
436 content = json.loads(body)
437 if not content:
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000438 logging.error('Unable to parse %s response from projecthosting.',
439 instance['name'])
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000440 return []
441
seanmccullough@chromium.org3e4a5812015-06-11 17:48:47 +0000442 issues = []
443 if 'items' in content:
444 items = content['items']
445 for item in items:
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +0000446 if instance.get('shorturl'):
Varun Khanejad9f97bc2017-08-02 12:55:01 -0700447 protocol = instance.get('short_url_protocol', 'http')
448 item_url = '%s://%s/%d' % (protocol, instance['shorturl'], item['id'])
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +0000449 else:
450 item_url = 'https://bugs.chromium.org/p/%s/issues/detail?id=%d' % (
451 instance['name'], item['id'])
seanmccullough@chromium.org3e4a5812015-06-11 17:48:47 +0000452 issue = {
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000453 'header': item['title'],
454 'created': dateutil.parser.parse(item['published']),
455 'modified': dateutil.parser.parse(item['updated']),
456 'author': item['author']['name'],
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +0000457 'url': item_url,
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000458 'comments': [],
459 'status': item['status'],
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +0000460 'labels': [],
461 'components': []
seanmccullough@chromium.org3e4a5812015-06-11 17:48:47 +0000462 }
463 if 'owner' in item:
464 issue['owner'] = item['owner']['name']
465 else:
466 issue['owner'] = 'None'
467 if issue['owner'] == user_str or issue['author'] == user_str:
468 issues.append(issue)
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +0000469 if 'labels' in item:
470 issue['labels'] = item['labels']
471 if 'components' in item:
472 issue['components'] = item['components']
seanmccullough@chromium.org3e4a5812015-06-11 17:48:47 +0000473
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000474 return issues
475
jsbell@chromium.orgc92f5822014-01-06 23:49:11 +0000476 def print_heading(self, heading):
477 print
478 print self.options.output_format_heading.format(heading=heading)
479
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000480 def match(self, author):
481 if '@' in self.user:
482 return author == self.user
483 return author.startswith(self.user + '@')
484
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000485 def print_change(self, change):
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000486 activity = len([
487 reply
488 for reply in change['replies']
489 if self.match(reply['author'])
490 ])
cjhopman@chromium.org53c1e562013-03-11 20:02:38 +0000491 optional_values = {
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000492 'created': change['created'].date().isoformat(),
493 'modified': change['modified'].date().isoformat(),
494 'reviewers': ', '.join(change['reviewers']),
495 'status': change['status'],
496 'activity': activity,
cjhopman@chromium.org53c1e562013-03-11 20:02:38 +0000497 }
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000498 if self.options.deltas:
499 optional_values['delta'] = change['delta']
500
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000501 self.print_generic(self.options.output_format,
502 self.options.output_format_changes,
503 change['header'],
504 change['review_url'],
cjhopman@chromium.org53c1e562013-03-11 20:02:38 +0000505 change['author'],
506 optional_values)
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000507
508 def print_issue(self, issue):
509 optional_values = {
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000510 'created': issue['created'].date().isoformat(),
511 'modified': issue['modified'].date().isoformat(),
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000512 'owner': issue['owner'],
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000513 'status': issue['status'],
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000514 }
515 self.print_generic(self.options.output_format,
516 self.options.output_format_issues,
517 issue['header'],
518 issue['url'],
519 issue['author'],
520 optional_values)
521
522 def print_review(self, review):
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000523 activity = len([
524 reply
525 for reply in review['replies']
526 if self.match(reply['author'])
527 ])
528 optional_values = {
529 'created': review['created'].date().isoformat(),
530 'modified': review['modified'].date().isoformat(),
531 'activity': activity,
532 }
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000533 self.print_generic(self.options.output_format,
534 self.options.output_format_reviews,
535 review['header'],
536 review['review_url'],
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000537 review['author'],
538 optional_values)
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000539
540 @staticmethod
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000541 def print_generic(default_fmt, specific_fmt,
542 title, url, author,
543 optional_values=None):
544 output_format = specific_fmt if specific_fmt is not None else default_fmt
545 output_format = unicode(output_format)
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000546 values = {
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000547 'title': title,
548 'url': url,
549 'author': author,
550 }
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000551 if optional_values is not None:
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000552 values.update(optional_values)
553 print DefaultFormatter().format(output_format, **values).encode(
554 sys.getdefaultencoding())
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000555
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000556
557 def filter_issue(self, issue, should_filter_by_user=True):
558 def maybe_filter_username(email):
559 return not should_filter_by_user or username(email) == self.user
560 if (maybe_filter_username(issue['author']) and
561 self.filter_modified(issue['created'])):
562 return True
563 if (maybe_filter_username(issue['owner']) and
564 (self.filter_modified(issue['created']) or
565 self.filter_modified(issue['modified']))):
566 return True
567 for reply in issue['replies']:
568 if self.filter_modified(reply['created']):
569 if not should_filter_by_user:
570 break
571 if (username(reply['author']) == self.user
572 or (self.user + '@') in reply['content']):
573 break
574 else:
575 return False
576 return True
577
578 def filter_modified(self, modified):
579 return self.modified_after < modified and modified < self.modified_before
580
581 def auth_for_changes(self):
582 #TODO(cjhopman): Move authentication check for getting changes here.
583 pass
584
585 def auth_for_reviews(self):
586 # Reviews use all the same instances as changes so no authentication is
587 # required.
588 pass
589
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000590 def get_changes(self):
591 for instance in rietveld_instances:
592 self.changes += self.rietveld_search(instance, owner=self.user)
593
594 for instance in gerrit_instances:
595 self.changes += self.gerrit_search(instance, owner=self.user)
596
597 def print_changes(self):
598 if self.changes:
jsbell@chromium.orgc92f5822014-01-06 23:49:11 +0000599 self.print_heading('Changes')
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000600 for change in self.changes:
601 self.print_change(change)
602
603 def get_reviews(self):
604 for instance in rietveld_instances:
605 self.reviews += self.rietveld_search(instance, reviewer=self.user)
606
607 for instance in gerrit_instances:
608 reviews = self.gerrit_search(instance, reviewer=self.user)
609 reviews = filter(lambda r: not username(r['owner']) == self.user, reviews)
610 self.reviews += reviews
611
612 def print_reviews(self):
613 if self.reviews:
jsbell@chromium.orgc92f5822014-01-06 23:49:11 +0000614 self.print_heading('Reviews')
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000615 for review in self.reviews:
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000616 self.print_review(review)
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000617
618 def get_issues(self):
619 for project in google_code_projects:
seanmccullough@chromium.org3e4a5812015-06-11 17:48:47 +0000620 self.issues += self.project_hosting_issue_search(project)
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000621
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000622 def print_issues(self):
623 if self.issues:
jsbell@chromium.orgc92f5822014-01-06 23:49:11 +0000624 self.print_heading('Issues')
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000625 for issue in self.issues:
626 self.print_issue(issue)
627
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000628 def print_activity(self):
629 self.print_changes()
630 self.print_reviews()
631 self.print_issues()
632
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +0000633 def dump_json(self, ignore_keys=None):
634 if ignore_keys is None:
635 ignore_keys = ['replies']
636
637 def format_for_json_dump(in_array):
638 output = {}
639 for item in in_array:
640 url = item.get('url') or item.get('review_url')
641 if not url:
642 raise Exception('Dumped item %s does not specify url' % item)
643 output[url] = dict(
644 (k, v) for k,v in item.iteritems() if k not in ignore_keys)
645 return output
646
647 class PythonObjectEncoder(json.JSONEncoder):
648 def default(self, obj): # pylint: disable=method-hidden
649 if isinstance(obj, datetime):
650 return obj.isoformat()
651 if isinstance(obj, set):
652 return list(obj)
653 return json.JSONEncoder.default(self, obj)
654
655 output = {
656 'reviews': format_for_json_dump(self.reviews),
657 'changes': format_for_json_dump(self.changes),
658 'issues': format_for_json_dump(self.issues)
659 }
660 print json.dumps(output, indent=2, cls=PythonObjectEncoder)
661
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000662
663def main():
664 # Silence upload.py.
665 rietveld.upload.verbosity = 0
666
667 parser = optparse.OptionParser(description=sys.modules[__name__].__doc__)
668 parser.add_option(
669 '-u', '--user', metavar='<email>',
670 default=os.environ.get('USER'),
671 help='Filter on user, default=%default')
672 parser.add_option(
673 '-b', '--begin', metavar='<date>',
wychen@chromium.org85cab632015-05-28 21:04:37 +0000674 help='Filter issues created after the date (mm/dd/yy)')
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000675 parser.add_option(
676 '-e', '--end', metavar='<date>',
wychen@chromium.org85cab632015-05-28 21:04:37 +0000677 help='Filter issues created before the date (mm/dd/yy)')
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000678 quarter_begin, quarter_end = get_quarter_of(datetime.today() -
679 relativedelta(months=2))
680 parser.add_option(
681 '-Q', '--last_quarter', action='store_true',
jsbell@chromium.org74bfde02014-04-09 17:14:54 +0000682 help='Use last quarter\'s dates, i.e. %s to %s' % (
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000683 quarter_begin.strftime('%Y-%m-%d'), quarter_end.strftime('%Y-%m-%d')))
684 parser.add_option(
685 '-Y', '--this_year', action='store_true',
686 help='Use this year\'s dates')
687 parser.add_option(
688 '-w', '--week_of', metavar='<date>',
wychen@chromium.org85cab632015-05-28 21:04:37 +0000689 help='Show issues for week of the date (mm/dd/yy)')
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000690 parser.add_option(
wychen@chromium.org8ba1ddb2015-04-29 00:04:25 +0000691 '-W', '--last_week', action='count',
692 help='Show last week\'s issues. Use more times for more weeks.')
jsbell@chromium.org74bfde02014-04-09 17:14:54 +0000693 parser.add_option(
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000694 '-a', '--auth',
695 action='store_true',
696 help='Ask to authenticate for instances with no auth cookie')
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000697 parser.add_option(
698 '-d', '--deltas',
699 action='store_true',
700 help='Fetch deltas for changes (slow).')
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000701
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000702 activity_types_group = optparse.OptionGroup(parser, 'Activity Types',
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000703 'By default, all activity will be looked up and '
704 'printed. If any of these are specified, only '
705 'those specified will be searched.')
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000706 activity_types_group.add_option(
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000707 '-c', '--changes',
708 action='store_true',
709 help='Show changes.')
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000710 activity_types_group.add_option(
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000711 '-i', '--issues',
712 action='store_true',
713 help='Show issues.')
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000714 activity_types_group.add_option(
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000715 '-r', '--reviews',
716 action='store_true',
717 help='Show reviews.')
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000718 parser.add_option_group(activity_types_group)
719
720 output_format_group = optparse.OptionGroup(parser, 'Output Format',
721 'By default, all activity will be printed in the '
722 'following format: {url} {title}. This can be '
723 'changed for either all activity types or '
724 'individually for each activity type. The format '
725 'is defined as documented for '
726 'string.format(...). The variables available for '
727 'all activity types are url, title and author. '
728 'Format options for specific activity types will '
729 'override the generic format.')
730 output_format_group.add_option(
731 '-f', '--output-format', metavar='<format>',
732 default=u'{url} {title}',
733 help='Specifies the format to use when printing all your activity.')
734 output_format_group.add_option(
735 '--output-format-changes', metavar='<format>',
736 default=None,
cjhopman@chromium.org53c1e562013-03-11 20:02:38 +0000737 help='Specifies the format to use when printing changes. Supports the '
738 'additional variable {reviewers}')
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000739 output_format_group.add_option(
740 '--output-format-issues', metavar='<format>',
741 default=None,
cjhopman@chromium.org53c1e562013-03-11 20:02:38 +0000742 help='Specifies the format to use when printing issues. Supports the '
743 'additional variable {owner}.')
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000744 output_format_group.add_option(
745 '--output-format-reviews', metavar='<format>',
746 default=None,
747 help='Specifies the format to use when printing reviews.')
jsbell@chromium.orgc92f5822014-01-06 23:49:11 +0000748 output_format_group.add_option(
749 '--output-format-heading', metavar='<format>',
750 default=u'{heading}:',
751 help='Specifies the format to use when printing headings.')
752 output_format_group.add_option(
753 '-m', '--markdown', action='store_true',
754 help='Use markdown-friendly output (overrides --output-format '
755 'and --output-format-heading)')
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +0000756 output_format_group.add_option(
757 '-j', '--json', action='store_true',
758 help='Output json data (overrides other format options)')
nyquist@chromium.org18bc90d2012-12-20 19:26:47 +0000759 parser.add_option_group(output_format_group)
vadimsh@chromium.orgcf6a5d22015-04-09 22:02:00 +0000760 auth.add_auth_options(parser)
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000761
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000762 parser.add_option(
763 '-v', '--verbose',
764 action='store_const',
765 dest='verbosity',
766 default=logging.WARN,
767 const=logging.INFO,
768 help='Output extra informational messages.'
769 )
770 parser.add_option(
771 '-q', '--quiet',
772 action='store_const',
773 dest='verbosity',
774 const=logging.ERROR,
775 help='Suppress non-error messages.'
776 )
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +0000777 parser.add_option(
778 '-o', '--output', metavar='<file>',
779 help='Where to output the results. By default prints to stdout.')
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000780
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000781 # Remove description formatting
782 parser.format_description = (
Quinten Yearsleyb2cc4a92016-12-15 13:53:26 -0800783 lambda _: parser.description) # pylint: disable=no-member
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000784
785 options, args = parser.parse_args()
786 options.local_user = os.environ.get('USER')
787 if args:
788 parser.error('Args unsupported')
789 if not options.user:
790 parser.error('USER is not set, please use -u')
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000791 options.user = username(options.user)
792
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000793 logging.basicConfig(level=options.verbosity)
794
795 # python-keyring provides easy access to the system keyring.
796 try:
797 import keyring # pylint: disable=unused-import,unused-variable,F0401
798 except ImportError:
799 logging.warning('Consider installing python-keyring')
800
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000801 if not options.begin:
802 if options.last_quarter:
803 begin, end = quarter_begin, quarter_end
804 elif options.this_year:
805 begin, end = get_year_of(datetime.today())
806 elif options.week_of:
807 begin, end = (get_week_of(datetime.strptime(options.week_of, '%m/%d/%y')))
jsbell@chromium.org74bfde02014-04-09 17:14:54 +0000808 elif options.last_week:
wychen@chromium.org8ba1ddb2015-04-29 00:04:25 +0000809 begin, end = (get_week_of(datetime.today() -
810 timedelta(days=1 + 7 * options.last_week)))
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000811 else:
812 begin, end = (get_week_of(datetime.today() - timedelta(days=1)))
813 else:
Daniel Cheng4b37ce62017-09-07 12:00:02 -0700814 begin = dateutil.parser.parse(options.begin)
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000815 if options.end:
Daniel Cheng4b37ce62017-09-07 12:00:02 -0700816 end = dateutil.parser.parse(options.end)
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000817 else:
818 end = datetime.today()
819 options.begin, options.end = begin, end
820
jsbell@chromium.orgc92f5822014-01-06 23:49:11 +0000821 if options.markdown:
822 options.output_format = ' * [{title}]({url})'
823 options.output_format_heading = '### {heading} ###'
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000824 logging.info('Searching for activity by %s', options.user)
825 logging.info('Using range %s to %s', options.begin, options.end)
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000826
827 my_activity = MyActivity(options)
828
829 if not (options.changes or options.reviews or options.issues):
830 options.changes = True
831 options.issues = True
832 options.reviews = True
833
834 # First do any required authentication so none of the user interaction has to
835 # wait for actual work.
836 if options.changes:
837 my_activity.auth_for_changes()
838 if options.reviews:
839 my_activity.auth_for_reviews()
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000840
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000841 logging.info('Looking up activity.....')
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000842
seanmccullough@chromium.org3e4a5812015-06-11 17:48:47 +0000843 try:
844 if options.changes:
845 my_activity.get_changes()
846 if options.reviews:
847 my_activity.get_reviews()
848 if options.issues:
849 my_activity.get_issues()
850 except auth.AuthenticationError as e:
Tobias Sargeantffb3c432017-03-08 14:09:14 +0000851 logging.error('auth.AuthenticationError: %s', e)
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000852
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +0000853 output_file = None
854 try:
855 if options.output:
856 output_file = open(options.output, 'w')
857 logging.info('Printing output to "%s"', options.output)
858 sys.stdout = output_file
859 except (IOError, OSError) as e:
Varun Khanejad9f97bc2017-08-02 12:55:01 -0700860 logging.error('Unable to write output: %s', e)
Nicolas Dossou-gbetee5deedf2017-03-15 16:26:47 +0000861 else:
862 if options.json:
863 my_activity.dump_json()
864 else:
865 my_activity.print_changes()
866 my_activity.print_reviews()
867 my_activity.print_issues()
868 finally:
869 if output_file:
870 logging.info('Done printing to file.')
871 sys.stdout = sys.__stdout__
872 output_file.close()
873
cjhopman@chromium.org04d119d2012-10-17 22:41:53 +0000874 return 0
875
876
877if __name__ == '__main__':
stevefung@chromium.org832d51e2015-05-27 12:52:51 +0000878 # Fix encoding to support non-ascii issue titles.
879 fix_encoding.fix_encoding()
880
sbc@chromium.org013731e2015-02-26 18:28:43 +0000881 try:
882 sys.exit(main())
883 except KeyboardInterrupt:
884 sys.stderr.write('interrupted\n')
885 sys.exit(1)