Futurize `mtstat` as much as possible
Downloading files now works on Python 2.7 again, though other things
still seem broken with various errors (though I'm not sure if they were
there before). I gave up on Python 3 compatibility, as the encoding
situation is just too confusing. (Reports are downloaded using the
`stubby` command, then extracted from its text proto output with
regexes, and then have to be turned back into binary again to be treated
as ZIP files. https://stackoverflow.com/a/24519338/1175455 is just a
taste of the problem.)
TEST=Run `python2.7 mtstat.py -d 10`, and check it succeeds. Some
invalid reports are to be expected.
BUG=none
Change-Id: Ia0eec0598f0ee8ab1ce203c370cc9748af4ab65d
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/mttools/+/2044826
Tested-by: Harry Cutts <hcutts@chromium.org>
Reviewed-by: Sean O'Brien <seobrien@chromium.org>
Commit-Queue: Harry Cutts <hcutts@chromium.org>
diff --git a/Makefile b/Makefile
index a9835fa..30f9ec1 100644
--- a/Makefile
+++ b/Makefile
@@ -54,7 +54,7 @@
# install symlinks to command line tools
ln -sf $(SRC)/mtedit.py /usr/bin/mtedit
ln -sf $(SRC)/mtreplay.py /usr/bin/mtreplay
- ln -sf $(SRC)/mtstat/main.py /usr/bin/mtstat
+ ln -sf $(SRC)/mtstat.py /usr/bin/mtstat
ln -sf $(SRC)/mtbringup/main.py /usr/bin/mtbringup
ln -sf $(SRC)/mtbringup/main_firmware.py /usr/bin/mtbringup-firmware
ln -sf $(SRC)/mtbringup/main_xorg.py /usr/bin/mtbringup-xorg
diff --git a/mtlib/feedback.py b/mtlib/feedback.py
index f7aa771..63fd741 100644
--- a/mtlib/feedback.py
+++ b/mtlib/feedback.py
@@ -10,6 +10,7 @@
import time
import getpass
import imghdr
+import math
import os
import os.path
import re
@@ -117,7 +118,8 @@
while True:
process = subprocess.Popen(cmd, shell=True,
stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
+ stderr=subprocess.PIPE,
+ universal_newlines=True)
output, errors = process.communicate()
errorcode = process.returncode
@@ -151,7 +153,8 @@
cmd = FeedbackDownloader.GAIA_CMD
process = subprocess.Popen(cmd, shell=True,
stdout=subprocess.PIPE,
- stderr=subprocess.PIPE)
+ stderr=subprocess.PIPE,
+ universal_newlines=True)
print("Authenticating...")
output, errors = process.communicate()
errorcode = process.returncode
@@ -165,19 +168,20 @@
return None
print("Done Authenticating")
- def DownloadIDs(self, num, end_time=None, page_token=''):
- if not end_time:
+ def DownloadIDs(self, num, end_time_ms=None, page_token=''):
+ if not end_time_ms:
dt = datetime.datetime.utcnow() - datetime.datetime(1970, 1, 1)
- end_time = (((dt.days * 24 * 60 * 60 + dt.seconds) * 1000) +
- (dt.microseconds / 10))
+ end_time_ms = (((dt.days * 24 * 60 * 60 + dt.seconds) * 1000) +
+ math.ceil(dt.microseconds / 10))
cmd = FeedbackDownloader.STUBBY_LIST_CMD.format(
- end_time=end_time, max_results=num, token=page_token)
+ end_time=int(end_time_ms), max_results=num, token=page_token)
output = self._StubbyCall(cmd)
- page_token = filter(lambda x: 'next_page_token' in x, output.split('\n'))[0]
+ lines = output.split('\n')
+ page_token = list(filter(lambda x: 'next_page_token' in x, lines))[0]
page_token = page_token.split(" ")[-1][1:-1]
- ids = filter(lambda x: 'id' in x, output.split('\n'))
+ ids = filter(lambda x: 'id' in x, lines)
ids = [x[7:-1] for x in ids]
return page_token, ids
diff --git a/mtlib/log.py b/mtlib/log.py
index de0db30..e398101 100644
--- a/mtlib/log.py
+++ b/mtlib/log.py
@@ -86,7 +86,8 @@
if self.evdev:
open(filename + '.evdev', 'w').write(self.evdev)
if self.system:
- open(filename + '.system', 'w').write(self.system)
+ open(filename + '.system', 'wb').write(
+ self.system.encode('utf-8', 'strict'))
if self.image:
open(filename + '.jpg', 'w').write(self.image)
diff --git a/mtstat/main.py b/mtstat.py
similarity index 98%
rename from mtstat/main.py
rename to mtstat.py
index 68a834b..0c72862 100755
--- a/mtstat/main.py
+++ b/mtstat.py
@@ -3,6 +3,8 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+from __future__ import absolute_import
+from __future__ import division
from __future__ import print_function
from mtedit import MTEdit
diff --git a/mtstat/__init__.py b/mtstat/__init__.py
index 1b953e4..75a1594 100644
--- a/mtstat/__init__.py
+++ b/mtstat/__init__.py
@@ -1 +1,3 @@
-from mtstat.mtstat import MTStat
+from __future__ import absolute_import
+
+from .mtstat import MTStat
diff --git a/mtstat/mtstat.py b/mtstat/mtstat.py
index aced96e..636d5ba 100755
--- a/mtstat/mtstat.py
+++ b/mtstat/mtstat.py
@@ -3,12 +3,14 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+from __future__ import absolute_import
+from __future__ import division
from __future__ import print_function
from collections import Counter, defaultdict, namedtuple, OrderedDict
from itertools import chain
from mtreplay import MTReplay
-from queryengine import Query, QueryEngine, QueryMatch, QueryResult
+from .queryengine import Query, QueryEngine, QueryMatch, QueryResult
import re
diff --git a/mtstat/queryengine.py b/mtstat/queryengine.py
index ab96b52..0d5ae2a 100755
--- a/mtstat/queryengine.py
+++ b/mtstat/queryengine.py
@@ -3,6 +3,8 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
+from __future__ import absolute_import
+from __future__ import division
from __future__ import print_function
from mtlib.log import FeedbackDownloader, FeedbackLog, Log
@@ -218,8 +220,6 @@
downloader = FeedbackDownloader()
dt = datetime.datetime.utcnow() - datetime.datetime(1970, 1, 1)
- end_time = (((dt.days * 24 * 60 * 60 + dt.seconds) * 1000) +
- (dt.microseconds / 10))
num_to_download = int(num_to_download)
num_downloaded = 0
invalid_ids = self.GetInvalidIDs()
@@ -229,7 +229,7 @@
# Download list of feedback report id's
num_this_iteration = min((num_to_download - num_downloaded) * 5, 500)
page_token, report_ids = downloader.DownloadIDs(
- num_this_iteration, end_time, page_token)
+ num_this_iteration, page_token=page_token)
# Download and check each report
parameters = [(r_id, downloader, invalid_ids) for r_id in report_ids]
@@ -259,7 +259,7 @@
""" Wrapper for subprocesses to run ExecuteSingle """
try:
return QueryEngine().ExecuteSingle(args[0], args[1])
- except Exception, e:
+ except Exception as e:
traceback.print_exc()
raise e
@@ -267,6 +267,6 @@
""" Wrapper for subprocesses to run DownloadFile """
try:
return QueryEngine().DownloadFile(args[0], args[1], args[2])
- except Exception, e:
+ except Exception as e:
traceback.print_exc()
raise e