Implement stage RPC which allows user to specify artifacts.
As specified in goto/devserver-cache, implement the stage RPC
and replace existing calls download/symbol/wait_for_status/stage_images
with their respective stage calls.
I'm supporting the old rpc's by mapping them to stage for now until
all callers use the new RPC. Advantage of using the new rpc means
less being downloaded (only download what you need rather than
everything).
This is a very large code change, sadly I can't really break it up that
much. Fortunately it's mostly deleting old/dead code.
If you haven't read the doc yet, please read it before reviewing (
http://goto.google.com/devserver-cache ). You can
pretty much review all the new code stand-alone without looking at what
we are replacing.
BUG=chromium-os:38427
TEST=Pylint + Unittests + download, wait_for_status, stage separately
using different combinations. Still continuing testing of debug symbols
etc.
Change-Id: I201dcdaa8a14024247eca222a8f2e47deea464b9
Reviewed-on: https://gerrit.chromium.org/gerrit/42401
Reviewed-by: Chris Sosa <sosa@chromium.org>
Tested-by: Chris Sosa <sosa@chromium.org>
diff --git a/gsutil_util_unittest.py b/gsutil_util_unittest.py
index 0822e5d..ca84970 100755
--- a/gsutil_util_unittest.py
+++ b/gsutil_util_unittest.py
@@ -15,6 +15,7 @@
import gsutil_util
+# pylint: disable=W0212
class GSUtilUtilTest(mox.MoxTestBase):
def setUp(self):
@@ -24,8 +25,6 @@
self._good_mock_process.returncode = 0
self._bad_mock_process = self.mox.CreateMock(subprocess.Popen)
self._bad_mock_process.returncode = 1
- self.mox.StubOutWithMock(time, 'sleep')
- time.sleep(mox.IgnoreArg()).MultipleTimes()
def _CallRunGS(self, str_should_contain, attempts=1):
"""Helper that wraps a RunGS for tests."""
@@ -46,6 +45,8 @@
def testDownloadFromGS(self):
"""Tests that we can run download build from gs with one error."""
+ self.mox.StubOutWithMock(time, 'sleep')
+ time.sleep(mox.IgnoreArg()).MultipleTimes()
self.mox.StubOutWithMock(subprocess, 'Popen', use_mock_anything=True)
# Make sure we our retry works.
@@ -56,6 +57,8 @@
def testDownloadFromGSButGSDown(self):
"""Tests that we fail correctly if we can't reach GS."""
+ self.mox.StubOutWithMock(time, 'sleep')
+ time.sleep(mox.IgnoreArg()).MultipleTimes()
self.mox.StubOutWithMock(subprocess, 'Popen', use_mock_anything=True)
self._CallRunGS('from to', attempts=gsutil_util.GSUTIL_ATTEMPTS + 1)
@@ -66,6 +69,99 @@
'from', 'to')
self.mox.VerifyAll()
+ def testGSNamesFromList(self):
+ """Test that we can detect whether the target artifacts are available."""
+ # Test when the all target files are available
+ pattern = '.*_full_.*'
+ uploaded_list = ['chromeos_R17-1413.0.0-a1_x86-mario_full_dev.bin',
+ 'debug.tgz',
+ 'autotest.tar.bz2']
+
+ names = gsutil_util._GetGSNamesFromList(uploaded_list, pattern)
+ self.assertEqual(names[0],
+ 'chromeos_R17-1413.0.0-a1_x86-mario_full_dev.bin')
+
+ bad_pattern = '_delta_'
+ # Test when some target files are missing
+ names = gsutil_util._GetGSNamesFromList(uploaded_list, bad_pattern)
+ self.assertEqual(names, [])
+
+ def testGetGSNamesWithWait(self):
+ """Test that we get the target artifact that is available."""
+ archive_url = ('gs://chromeos-image-archive/x86-mario-release/'
+ 'R17-1413.0.0-a1-b1346')
+ name = 'chromeos_R17-1413.0.0-a1_x86-mario_full_dev.bin'
+ pattern = '_full_'
+ mock_data = 'mock data\nmock_data\nmock_data'
+ msg = 'UNIT TEST'
+
+ self.mox.StubOutWithMock(gsutil_util, 'GSUtilRun')
+ self.mox.StubOutWithMock(gsutil_util, '_GetGSNamesFromList')
+
+ # GSUtil cat gs://archive_url_prefix/UPLOADED.
+ gsutil_util.GSUtilRun(mox.StrContains(gsutil_util.UPLOADED_LIST),
+ mox.IgnoreArg()).AndReturn(mock_data)
+ gsutil_util._GetGSNamesFromList(mock_data.split('\n'),
+ pattern).AndReturn([name])
+
+ self.mox.ReplayAll()
+ returned_names = gsutil_util.GetGSNamesWithWait(
+ pattern, archive_url, msg, delay=1)
+ self.assertEqual([name], returned_names)
+ self.mox.VerifyAll()
+
+ def testGetGSNamesWithWaitWithRetry(self):
+ """Test that we can poll until all target artifacts are available."""
+ archive_url = ('gs://chromeos-image-archive/x86-mario-release/'
+ 'R17-1413.0.0-a1-b1346')
+ name = 'chromeos_R17-1413.0.0-a1_x86-mario_full_dev.bin'
+ pattern = '_full_'
+ mock_data = 'mock data\nmock_data\nmock_data'
+ msg = 'UNIT TEST'
+
+ self.mox.StubOutWithMock(gsutil_util, 'GSUtilRun')
+ self.mox.StubOutWithMock(gsutil_util, '_GetGSNamesFromList')
+
+ # GSUtil cat gs://archive_url_prefix/UPLOADED.
+ gsutil_util.GSUtilRun(mox.StrContains(gsutil_util.UPLOADED_LIST),
+ mox.IgnoreArg()).AndReturn(mock_data)
+ gsutil_util._GetGSNamesFromList(mock_data.split('\n'),
+ pattern).AndReturn(None)
+
+ gsutil_util.GSUtilRun(mox.StrContains(gsutil_util.UPLOADED_LIST),
+ mox.IgnoreArg()).AndReturn(mock_data)
+ gsutil_util._GetGSNamesFromList(mox.IgnoreArg(),
+ mox.IgnoreArg()).AndReturn([name])
+
+ self.mox.ReplayAll()
+ returned_names = gsutil_util.GetGSNamesWithWait(
+ pattern, archive_url, msg, delay=1)
+ self.assertEqual(name, returned_names[0])
+ self.mox.VerifyAll()
+
+ def testGetGSNamesWithWaitTimeout(self):
+ """Test that we wait for the target artifacts until timeout occurs."""
+ archive_url = ('gs://chromeos-image-archive/x86-mario-release/'
+ 'R17-1413.0.0-a1-b1346')
+ pattern = '_full_'
+ mock_data = 'mock data\nmock_data\nmock_data'
+ msg = 'UNIT TEST'
+
+ self.mox.StubOutWithMock(gsutil_util, 'GSUtilRun')
+ self.mox.StubOutWithMock(gsutil_util, '_GetGSNamesFromList')
+
+ # GSUtil cat gs://archive_url_prefix/UPLOADED.
+ gsutil_util.GSUtilRun(mox.StrContains(gsutil_util.UPLOADED_LIST),
+ mox.IgnoreArg()).AndReturn(mock_data)
+ gsutil_util._GetGSNamesFromList(mock_data.split('\n'),
+ pattern).AndReturn(None)
+
+ self.mox.ReplayAll()
+ returned_name = gsutil_util.GetGSNamesWithWait(
+ pattern, archive_url, msg, delay=2, timeout=1)
+ self.assertEqual(returned_name, None)
+ self.mox.VerifyAll()
+
if __name__ == '__main__':
unittest.main()